feat(conf+builder): build without contrib/typescript

89c8f9767a1ef610f9ee050e1a5da5728bba02d7

build/conf/ts/node_modules.conf

@@ -9,6 +9,10 @@ macro CUSTOM_CONTRIB_TYPESCRIPT(P) {
     SET(NPM_CONTRIBS_PATH $P)
 }
 
+macro NO_CONTRIB_TYPESCRIPT() {
+    SET(NPM_CONTRIBS_PATH -)
+}
+
 ### @usage: NPM_CONTRIBS() # internal
 ###
 ### Defines special module that provides contrib tarballs from internal npm registry.
@@ -55,10 +59,14 @@ macro _TS_ADD_NODE_MODULES_FOR_BUILDER() {
 
 _TARBALLS_STORE=__tarballs__
 _PREPARE_DEPS_INOUTS=
+_PREPARE_DEPS_RESOURCES=
+_PREPARE_DEPS_USE_RESOURCES_FLAG=
 _PREPARE_DEPS_CMD=$TOUCH_UNIT \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS prepare-deps \
     --tarballs-store $_TARBALLS_STORE \
     $_PREPARE_DEPS_INOUTS \
+    $_PREPARE_DEPS_RESOURCES \
+    $_PREPARE_DEPS_USE_RESOURCES_FLAG \
     ${kv;hide:"pc magenta"} ${kv;hide:"p TS_DEP"}
 
 # In case of no deps we need to create empty outputs for graph connectivity

build/plugins/lib/nots/package_manager/base/lockfile.py

@@ -29,6 +29,10 @@ class LockfilePackageMeta(object):
     def to_str(self):
         return " ".join([self.tarball_url, self.sky_id, self.integrity, self.integrity_algorithm])
 
+    def to_uri(self):
+        pkg_uri = f"{self.tarball_url}#integrity={self.integrity_algorithm}-{self.integrity}"
+        return pkg_uri
+
 
 class LockfilePackageMetaInvalidError(RuntimeError):
     pass

build/plugins/lib/nots/package_manager/pnpm/lockfile.py

@@ -1,5 +1,4 @@
 import base64
-import binascii
 import yaml
 import os
 import io
@@ -177,11 +176,16 @@ def _parse_package_integrity(integrity):
     """
     algo, hash_b64 = integrity.split("-", 1)
 
+    if algo not in ("sha1", "sha512"):
+        raise LockfilePackageMetaInvalidError(
+            f"Invalid package integrity algorithm, expected one of ('sha1', 'sha512'), got '{algo}'"
+        )
+
     try:
-        hash_hex = binascii.hexlify(base64.b64decode(hash_b64))
+        base64.b64decode(hash_b64)
     except TypeError as e:
         raise LockfilePackageMetaInvalidError(
             "Invalid package integrity encoding, integrity: {}, error: {}".format(integrity, e)
         )
 
-    return (algo, hash_hex)
+    return (algo, hash_b64)

build/plugins/lib/nots/package_manager/pnpm/package_manager.py

@@ -87,6 +87,30 @@ class PnpmPackageManager(BasePackageManager):
                 bundle_path=os.path.join(self.build_path, NODE_MODULES_WORKSPACE_BUNDLE_FILENAME),
             )
 
+    def calc_prepare_deps_inouts_and_resources(
+        self, store_path: str, has_deps: bool
+    ) -> tuple[list[str], list[str], list[str]]:
+        ins = [
+            s_rooted(build_pj_path(self.module_path)),
+            s_rooted(build_lockfile_path(self.module_path)),
+        ]
+        outs = [
+            b_rooted(build_ws_config_path(self.module_path)),
+            b_rooted(build_pre_lockfile_path(self.module_path)),
+        ]
+        resources = []
+
+        if has_deps:
+            for dep_path in self.get_local_peers_from_package_json():
+                ins.append(b_rooted(build_ws_config_path(dep_path)))
+                ins.append(b_rooted(build_pre_lockfile_path(dep_path)))
+
+            for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]):
+                resources.append(pkg.to_uri())
+                outs.append(b_rooted(self._tarballs_store_path(pkg, store_path)))
+
+        return ins, outs, resources
+
     # TODO: FBP-1254
     # def calc_prepare_deps_inouts(self, store_path: str, has_deps: bool) -> (list[str], list[str]):
     def calc_prepare_deps_inouts(self, store_path, has_deps):

build/plugins/lib/nots/package_manager/pnpm/tests/test_lockfile.py

@@ -99,10 +99,7 @@ def test_lockfile_get_packages_meta_ok():
     assert len(packages) == 1
     assert pkg.tarball_url == "@babel%2fcli/-/cli-7.6.2.tgz"
     assert pkg.sky_id == "rbtorrent:cb1849da3e4947e56a8f6bde6a1ec42703ddd187"
-    assert (
-        pkg.integrity
-        == b"24367e4ff6ebf693df4f696600c272a490d34d31ccf5e3c3fc40f5d13463473255744572f89077891961cd8993b796243601efc561a55159cbb5dbfaaee883ad"
-    )
+    assert pkg.integrity == "JDZ+T/br9pPfT2lmAMJypJDTTTHM9ePD/ED10TRjRzJVdEVy+JB3iRlhzYmTt5YkNgHvxWGlUVnLtdv6ruiDrQ=="
     assert pkg.integrity_algorithm == "sha512"
 
 

build/plugins/nots.py

@@ -542,20 +542,39 @@ def _select_matching_version(erm_json, resource_name, range_str, dep_is_required
 
 @_with_report_configure_error
 def on_prepare_deps_configure(unit):
-    # Originally this peerdir was in .conf file
-    # but it kept taking default value of NPM_CONTRIBS_PATH
-    # before it was updated by CUSTOM_CONTRIB_TYPESCRIPT()
-    # so I moved it here.
-    unit.onpeerdir(unit.get("NPM_CONTRIBS_PATH"))
+    contrib_path = unit.get("NPM_CONTRIBS_PATH")
+    if contrib_path == '-':
+        unit.on_prepare_deps_configure_no_contrib()
+        return
+    unit.onpeerdir(contrib_path)
     pm = _create_pm(unit)
     pj = pm.load_package_json_from_dir(pm.sources_path)
     has_deps = pj.has_dependencies()
     ins, outs = pm.calc_prepare_deps_inouts(unit.get("_TARBALLS_STORE"), has_deps)
 
-    if pj.has_dependencies():
+    if has_deps:
+        unit.onpeerdir(pm.get_local_peers_from_package_json())
+        __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
+        __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
+
+    else:
+        __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", [], sorted(outs)))
+        unit.set(["_PREPARE_DEPS_CMD", "$_PREPARE_NO_DEPS_CMD"])
+
+
+@_with_report_configure_error
+def on_prepare_deps_configure_no_contrib(unit):
+    pm = _create_pm(unit)
+    pj = pm.load_package_json_from_dir(pm.sources_path)
+    has_deps = pj.has_dependencies()
+    ins, outs, resources = pm.calc_prepare_deps_inouts_and_resources(unit.get("_TARBALLS_STORE"), has_deps)
+
+    if has_deps:
         unit.onpeerdir(pm.get_local_peers_from_package_json())
         __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
         __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
+        unit.set(["_PREPARE_DEPS_RESOURCES", " ".join([f'${{resource:"{uri}"}}' for uri in sorted(resources)])])
+        unit.set(["_PREPARE_DEPS_USE_RESOURCES_FLAG", "--resource-root $(RESOURCE_ROOT)"])
 
     else:
         __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", [], sorted(outs)))

build/scripts/fetch_from_npm.py

@@ -4,6 +4,7 @@ import time
 import logging
 import argparse
 import hashlib
+import base64
 
 import sky
 import fetch_from
@@ -77,11 +78,12 @@ def _fetch_via_http(tarball_url, integrity, integrity_algorithm, file_name):
 
     hashobj = hashlib.new(integrity_algorithm)
     fetched_file = fetch_from.fetch_url(url, False, file_name, tries=1, writers=[hashobj.update])
+    checksum = base64.b64encode(hashobj.digest()).decode('utf-8')
 
-    if hashobj.hexdigest() != integrity:
+    if checksum != integrity:
         raise fetch_from.BadChecksumFetchError("Expected {}, but got {} for {}".format(
             integrity,
-            hashobj.hexdigest(),
+            checksum,
             file_name,
         ))