diff --git a/npm/private/BUILD.bazel b/npm/private/BUILD.bazel index 117f037c5..3efc7c60d 100644 --- a/npm/private/BUILD.bazel +++ b/npm/private/BUILD.bazel @@ -196,6 +196,7 @@ bzl_library( ":npm_translate_lock_macro_helpers", ":npm_translate_lock_state", ":npmrc", + ":pnpm", ":pnpm_repository", ":transitive_closure", ":utils", @@ -218,6 +219,12 @@ bzl_library( ] + (["@bazel_tools//tools/build_defs/repo:cache.bzl"] if bazel_lib_utils.is_bazel_7_or_greater() else []), ) +bzl_library( + name = "pnpm", + srcs = ["pnpm.bzl"], + visibility = ["//npm:__subpackages__"], +) + bzl_library( name = "transitive_closure", srcs = ["transitive_closure.bzl"], diff --git a/npm/private/npm_translate_lock_helpers.bzl b/npm/private/npm_translate_lock_helpers.bzl index 9f3e5ad5c..acc99c158 100644 --- a/npm/private/npm_translate_lock_helpers.bzl +++ b/npm/private/npm_translate_lock_helpers.bzl @@ -259,7 +259,7 @@ def _get_npm_imports(importers, packages, patched_dependencies, only_built_depen # special case for alias dependencies such as npm:alias-to@version maybe_package = dep_version[4:] elif dep_version not in packages: - maybe_package = utils.pnpm_name(dep_package, dep_version) + maybe_package = utils.package_key(dep_package, dep_version) else: maybe_package = dep_version if maybe_package not in linked_packages: diff --git a/npm/private/npm_translate_lock_state.bzl b/npm/private/npm_translate_lock_state.bzl index b944cb156..e3a3cf448 100644 --- a/npm/private/npm_translate_lock_state.bzl +++ b/npm/private/npm_translate_lock_state.bzl @@ -7,6 +7,7 @@ load("@bazel_skylib//lib:dicts.bzl", "dicts") load("@bazel_skylib//lib:paths.bzl", "paths") load(":npm_translate_lock_helpers.bzl", "helpers") load(":npmrc.bzl", "parse_npmrc") +load(":pnpm.bzl", "pnpm") load(":repository_label_store.bzl", "repository_label_store") load(":utils.bzl", "INTERNAL_ERROR_MSG", "utils") @@ -490,7 +491,7 @@ def _load_lockfile(priv, rctx, _, label_store): if result.return_code: lock_parse_err = "failed to parse pnpm lock file with yq. '{}' exited with {}: \nSTDOUT:\n{}\nSTDERR:\n{}".format(" ".join(yq_args), result.return_code, result.stdout, result.stderr) else: - importers, packages, patched_dependencies, lock_version, lock_parse_err = utils.parse_pnpm_lock_json(result.stdout if result.stdout != "null" else None) # NB: yq will return the string "null" if the yaml file is empty + importers, packages, patched_dependencies, lock_version, lock_parse_err = pnpm.parse_pnpm_lock_json(result.stdout if result.stdout != "null" else None) # NB: yq will return the string "null" if the yaml file is empty priv["lock_version"] = lock_version priv["importers"] = importers diff --git a/npm/private/pnpm.bzl b/npm/private/pnpm.bzl new file mode 100644 index 000000000..884703219 --- /dev/null +++ b/npm/private/pnpm.bzl @@ -0,0 +1,592 @@ +"Pnpm lockfile parsing and conversion to rules_js format." + +load("@bazel_skylib//lib:types.bzl", "types") +load(":utils.bzl", "DEFAULT_REGISTRY_DOMAIN_SLASH", "utils") + +def _is_vendored_tarfile(package_snapshot): + if "resolution" in package_snapshot: + return "tarball" in package_snapshot["resolution"] + return False + +def _to_package_key(name, version): + if not version[0].isdigit(): + return version + return "{}@{}".format(name, version) + +# Metadata about a pnpm "project" (importer). +# +# Metadata may come from different locations depending on the lockfile, this struct should +# have data normalized across lockfiles. +def _new_import_info(dependencies, dev_dependencies, optional_dependencies): + return { + "dependencies": dependencies, + "dev_dependencies": dev_dependencies, + "optional_dependencies": optional_dependencies, + } + +# Metadata about a package. +# +# Metadata may come from different locations depending on the lockfile, this struct should +# have data normalized across lockfiles. +def _new_package_info(id, name, dependencies, optional_dependencies, peer_dependencies, dev, has_bin, optional, requires_build, version, friendly_version, resolution): + return { + "id": id, + "name": name, + "dependencies": dependencies, + "optional_dependencies": optional_dependencies, + "peer_dependencies": peer_dependencies, + "dev": dev, + "has_bin": has_bin, + "optional": optional, + "requires_build": requires_build, + "version": version, + "friendly_version": friendly_version, + "resolution": resolution, + } + +######################### Lockfile v5.4 ######################### + +def _strip_v5_v6_default_registry(name_version): + # Strip the default registry from the name_version string + if name_version.startswith(DEFAULT_REGISTRY_DOMAIN_SLASH): + return name_version[len(DEFAULT_REGISTRY_DOMAIN_SLASH):] + return name_version + +def _convert_v5_v6_file_package(package_path, package_snapshot): + if "name" not in package_snapshot: + msg = "expected package {} to have a name field".format(package_path) + fail(msg) + + name = package_snapshot["name"] + version = package_path + if _is_vendored_tarfile(package_snapshot): + if "version" in package_snapshot: + version = package_snapshot["version"] + friendly_version = version + else: + friendly_version = package_snapshot["version"] if "version" in package_snapshot else version + + return name, version, friendly_version + +def _strip_v5_peer_dep_or_patched_version(version): + "Remove peer dependency or patched syntax from version string" + + # 21.1.0_rollup@2.70.2 becomes 21.1.0 + # 1.0.0_o3deharooos255qt5xdujc3cuq becomes 1.0.0 + index = version.find("_") + if index != -1: + return version[:index] + return version + +def _strip_v5_default_registry_to_version(name, version): + # Strip the default registry/name/ from the version string + pre = DEFAULT_REGISTRY_DOMAIN_SLASH + name + "/" + if version.startswith(pre): + return version[len(pre):] + return version + +def _convert_v5_importer_dependency_map(specifiers, deps): + result = {} + for name, version in deps.items(): + specifier = specifiers.get(name) + + if specifier.startswith("npm:"): + # Keep the npm: specifier for aliased dependencies + # convert v5 style aliases ([default_registry]/aliased/version) to npm:aliased@version + alias, version = _strip_v5_v6_default_registry(version).lstrip("/").rsplit("/", 1) + version = "npm:{}@{}".format(alias, version) + else: + # Transition [registry/]name/version[_patch][_peer_data] to a rules_js version format + version = _convert_pnpm_v5_version_peer_dep(_strip_v5_default_registry_to_version(name, version)) + + result[name] = version + return result + +def _convert_v5_importers(importers): + result = {} + for import_path, importer in importers.items(): + specifiers = importer.get("specifiers", {}) + + result[import_path] = _new_import_info( + dependencies = _convert_v5_importer_dependency_map(specifiers, importer.get("dependencies", {})), + dev_dependencies = _convert_v5_importer_dependency_map(specifiers, importer.get("devDependencies", {})), + optional_dependencies = _convert_v5_importer_dependency_map(specifiers, importer.get("optionalDependencies", {})), + ) + return result + +def _convert_pnpm_v5_version_peer_dep(version): + # Covert a pnpm lock file v5 version string of the format + # 1.2.3_@scope+peer@2.0.2_@scope+peer@4.5.6 + # to a version_peer_version that is compatible with rules_js. + + # If there is a suffix to the version + peer_dep_index = version.find("_") + if peer_dep_index != -1: + # if the suffix contains an @version (not just a _patchhash) + peer_dep_index = version.find("@", peer_dep_index) + if peer_dep_index != -1: + peer_dep = version[peer_dep_index:] + version = version[0:peer_dep_index] + utils.sanitize_string(peer_dep) + + return version + +def _convert_pnpm_v5_package_dependency_version(name, version): + # an alias to an alternate package + if version.startswith("/"): + alias, version = version[1:].rsplit("/", 1) + return "npm:{}@{}".format(alias, version) + + # Removing the default registry+name from the version string + version = _strip_v5_default_registry_to_version(name, version) + + # Convert peer dependency data to rules_js ~v5 format + version = _convert_pnpm_v5_version_peer_dep(version) + + return version + +def _convert_pnpm_v5_package_dependency_map(deps): + result = {} + for name, version in deps.items(): + result[name] = _convert_pnpm_v5_package_dependency_version(name, version) + return result + +def _convert_v5_packages(packages): + result = {} + for package_path, package_snapshot in packages.items(): + if "resolution" not in package_snapshot: + msg = "package {} has no resolution field".format(package_path) + fail(msg) + + package_path = _convert_pnpm_v5_version_peer_dep(package_path) + + if package_path.startswith("file:"): + # direct reference to file + name, version, friendly_version = _convert_v5_v6_file_package(package_path, package_snapshot) + elif "name" in package_snapshot and "version" in package_snapshot: + # key/path is complicated enough the real name+version are properties + name = package_snapshot["name"] + version = _strip_v5_default_registry_to_version(name, package_path) + friendly_version = package_snapshot["version"] + elif package_path.startswith("/"): + # a simple /name/version[_peer_info] + name, version = package_path[1:].rsplit("/", 1) + friendly_version = _strip_v5_peer_dep_or_patched_version(version) + else: + msg = "unexpected package path: {} of {}".format(package_path, package_snapshot) + fail(msg) + + package_key = _to_package_key(name, version) + + package_info = _new_package_info( + id = package_snapshot.get("id", None), + name = name, + version = version, + friendly_version = friendly_version, + dependencies = _convert_pnpm_v5_package_dependency_map(package_snapshot.get("dependencies", {})), + peer_dependencies = _convert_pnpm_v5_package_dependency_map(package_snapshot.get("peerDependencies", {})), + optional_dependencies = _convert_pnpm_v5_package_dependency_map(package_snapshot.get("optionalDependencies", {})), + dev = package_snapshot.get("dev", False), + has_bin = package_snapshot.get("hasBin", False), + optional = package_snapshot.get("optional", False), + requires_build = package_snapshot.get("requiresBuild", False), + resolution = package_snapshot.get("resolution"), + ) + + if package_key in result: + msg = "WARNING: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) + + # buildifier: disable=print + print(msg) + + result[package_key] = package_info + return result + +######################### Lockfile v6 ######################### + +def _convert_pnpm_v6_v9_version_peer_dep(version): + # Covert a pnpm lock file v6 version string of the format + # 1.2.3(@scope/peer@2.0.2)(@scope/peer@4.5.6) + # to a version_peer_version that is compatible with rules_js. + if version[-1] == ")": + # Drop the patch_hash= not present in v5 so (patch_hash=123) -> (123) like v5 + version = version.replace("(patch_hash=", "(") + + # There is a peer dep if the string ends with ")" + peer_dep_index = version.find("(") + peer_dep = version[peer_dep_index:] + if len(peer_dep) > 32: + # Prevent long paths. The pnpm lockfile v6 no longer hashes long sequences of + # peer deps so we must hash here to prevent extremely long file paths that lead to + # "File name too long) build failures. + peer_dep = "_" + utils.hash(peer_dep) + version = version[0:peer_dep_index] + utils.sanitize_string(peer_dep) + version = version.rstrip("_") + return version + +def _strip_v6_default_registry_to_version(name, version): + # Strip the default registry/name@ from the version string + pre = DEFAULT_REGISTRY_DOMAIN_SLASH + name + "@" + if version.startswith(pre): + return version[len(pre):] + return version + +def _convert_pnpm_v6_importer_dependency_map(deps): + result = {} + for name, attributes in deps.items(): + specifier = attributes.get("specifier") + version = attributes.get("version") + + if specifier.startswith("npm:"): + # Keep the npm: specifier for aliased dependencies + # convert v6 style aliases ([registry]/aliased@version) to npm:aliased@version + alias, version = _strip_v5_v6_default_registry(version).lstrip("/").rsplit("@", 1) + version = "npm:{}@{}".format(alias, version) + else: + # Transition [registry/]name@version[(peer)(data)] to a rules_js version format + version = _convert_pnpm_v6_v9_version_peer_dep(_strip_v6_default_registry_to_version(name, version)) + + result[name] = version + return result + +def _convert_v6_importers(importers): + # Convert pnpm lockfile v6 importers to a rules_js compatible ~v5 format. + # + # v5 importers: + # specifiers: + # pkg-a: 1.2.3 + # pkg-b: ^4.5.6 + # deps: + # pkg-a: 1.2.3 + # devDeps: + # pkg-b: 4.10.1 + # ... + # + # v6 pushed the 'specifiers' and 'version' into subproperties: + # + # deps: + # pkg-a: + # specifier: 1.2.3 + # version: 1.2.3 + # devDeps: + # pkg-b: + # specifier: ^4.5.6 + # version: 4.10.1 + + result = {} + for import_path, importer in importers.items(): + result[import_path] = _new_import_info( + dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("dependencies", {})), + dev_dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("devDependencies", {})), + optional_dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("optionalDependencies", {})), + ) + return result + +def _convert_pnpm_v6_package_dependency_version(name, version): + # an alias to an alternate package + if version.startswith("/"): + # Convert peer dependency data to rules_js ~v5 format + version = _convert_pnpm_v6_v9_version_peer_dep(version[1:]) + + return "npm:{}".format(version) + + # Removing the default registry+name from the version string + version = _strip_v6_default_registry_to_version(name, version) + + # Convert peer dependency data to rules_js ~v5 format + version = _convert_pnpm_v6_v9_version_peer_dep(version) + + return version + +def _convert_pnpm_v6_package_dependency_map(deps): + result = {} + for name, version in deps.items(): + result[name] = _convert_pnpm_v6_package_dependency_version(name, version) + return result + +def _convert_v6_packages(packages): + # Convert pnpm lockfile v6 importers to a rules_js compatible ~v5 format. + # + # v6 package metadata mainly changed formatting of metadata such as: + # + # dependency versions with peers: + # v5: 2.0.0_@aspect-test+c@2.0.2 + # v6: 2.0.0(@aspect-test/c@2.0.2) + + result = {} + for package_path, package_snapshot in packages.items(): + if "resolution" not in package_snapshot: + msg = "package {} has no resolution field".format(package_path) + fail(msg) + + package_path = _convert_pnpm_v6_v9_version_peer_dep(package_path) + + if package_path.startswith("file:"): + # direct reference to file + name, version, friendly_version = _convert_v5_v6_file_package(package_path, package_snapshot) + elif "name" in package_snapshot and "version" in package_snapshot: + # key/path is complicated enough the real name+version are properties + name = package_snapshot["name"] + version = _strip_v6_default_registry_to_version(name, package_path) + friendly_version = package_snapshot["version"] + elif package_path.startswith("/"): + # plain /pkg@version(_peer_info) + name, version = package_path[1:].rsplit("@", 1) + friendly_version = _strip_v5_peer_dep_or_patched_version(version) # NOTE: already converted to v5 peer style + else: + msg = "unexpected package path: {} of {}".format(package_path, package_snapshot) + fail(msg) + + package_key = _to_package_key(name, version) + + package_info = _new_package_info( + id = package_snapshot.get("id", None), + name = name, + version = version, + friendly_version = friendly_version, + dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("dependencies", {})), + peer_dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("peerDependencies", {})), + optional_dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("optionalDependencies", {})), + dev = package_snapshot.get("dev", False), + has_bin = package_snapshot.get("hasBin", False), + optional = package_snapshot.get("optional", False), + requires_build = package_snapshot.get("requiresBuild", False), + resolution = package_snapshot.get("resolution"), + ) + + if package_key in result: + msg = "ERROR: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) + + # buildifier: disable=print + print(msg) + + result[package_key] = package_info + + return result + +######################### Lockfile v9 ######################### + +def _convert_pnpm_v9_package_dependency_version(snapshots, name, version): + # Detect when an alias is just a direct reference to another snapshot + is_alias = version in snapshots + + # Convert peer dependency data to rules_js ~v5 format + version = _convert_pnpm_v6_v9_version_peer_dep(version) + + return "npm:{}".format(version) if is_alias else version + +def _convert_pnpm_v9_package_dependency_map(snapshots, deps): + result = {} + for name, version in deps.items(): + result[name] = _convert_pnpm_v9_package_dependency_version(snapshots, name, version) + return result + +def _convert_pnpm_v9_importer_dependency_map(deps): + result = {} + for name, attributes in deps.items(): + specifier = attributes.get("specifier") + version = attributes.get("version") + + # Transition version[(patch)(peer)(data)] to a rules_js version format + version = _convert_pnpm_v6_v9_version_peer_dep(version) + + if specifier.startswith("npm:"): + # Keep the npm: specifier for aliased dependencies + alias, version = version.rsplit("@", 1) + version = "npm:{}@{}".format(alias, version) + + result[name] = version + return result + +def _convert_v9_importers(importers): + # Convert pnpm lockfile v9 importers to a rules_js compatible ~v5 format. + # Almost identical to v6 but with fewer odd edge cases. + + result = {} + for import_path, importer in importers.items(): + result[import_path] = _new_import_info( + dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("dependencies", {})), + dev_dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("devDependencies", {})), + optional_dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("optionalDependencies", {})), + ) + return result + +def _convert_v9_packages(packages, snapshots): + # Convert pnpm lockfile v9 importers to a rules_js compatible format. + + # v9 split package metadata (v6 "packages" field) into 2: + # + # The 'snapshots' keys contain the resolved dependencies such as each unique combo of deps/peers/versions + # while 'packages' contain the static information about each and every package@version such as hasBin, + # resolution and static dep data. + # + # Note all static registry info such as URLs has moved from the 'importers[x/pkg@version].version' and 'packages[x/pkg@version]' to + # only being present in the actual packages[pkg@version].resolution.* + # + # Example: + # + # packages: + # '@scoped/name@5.0.2' + # hasBin + # resolution (registry-url, integrity etc) + # peerDependencies which *might* be resolved + # + # snapshots: + # pkg@http://a/url + # ... + # + # '@scoped/name@2.0.0(peer@2.0.2)' + # dependencies: + # a-dep: 1.2.3 + # peer: 2.0.2 + # b-dep: 3.2.1(peer-b@4.5.6) + # alias: actual@1.2.3 + # l: file:../path/to/dir + # x: https://a/url/v1.2.3.tar.gz + + result = {} + + # Snapshots contains the packages with the keys (which include peers) to return + for package_key, package_snapshot in snapshots.items(): + peer_meta_index = package_key.find("(") + static_key = package_key[:peer_meta_index] if peer_meta_index > 0 else package_key + if not static_key in packages: + msg = "package {} not found in pnpm 'packages'".format(static_key) + fail(msg) + + package_data = packages[static_key] + + if "resolution" not in package_data: + msg = "package {} has no resolution field".format(static_key) + fail(msg) + + # the raw name + version are the key, not including peerDeps+patch + version_index = static_key.index("@", 1) + name = static_key[:version_index] + package_key = _convert_pnpm_v6_v9_version_peer_dep(package_key) + + # Extract the version including peerDeps+patch from the key + version = package_key[package_key.index("@", 1) + 1:] + + # package_data can have the resolved "version" for things like https:// deps + friendly_version = package_data["version"] if "version" in package_data else static_key[version_index + 1:] + + package_info = _new_package_info( + id = package_data.get("id", None), # TODO: does v9 have "id"? + name = name, + version = version, + friendly_version = friendly_version, + dependencies = _convert_pnpm_v9_package_dependency_map(snapshots, package_snapshot.get("dependencies", {})), + optional_dependencies = _convert_pnpm_v9_package_dependency_map(snapshots, package_snapshot.get("optionalDependencies", {})), + peer_dependencies = package_data.get("peerDependencies", {}), + dev = None, # TODO(pnpm9): must inspect importers.*.devDependencies? + has_bin = package_data.get("hasBin", False), + optional = package_snapshot.get("optional", False), + requires_build = None, # Unknown from lockfile in v9 + resolution = package_data.get("resolution"), + ) + + if package_key in result: + msg = "ERROR: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) + fail(msg) + + result[package_key] = package_info + + return result + +######################### Pnpm API ######################### + +def _parse_pnpm_lock_json(content): + """Parse the content of a pnpm-lock.yaml file. + + Args: + content: lockfile content as json + + Returns: + A tuple of (importers dict, packages dict, patched_dependencies dict, error string) + """ + return _parse_lockfile(json.decode(content) if content else None, None) + +def _parse_lockfile(parsed, err): + """Helper function used by _parse_pnpm_lock_json. + + Args: + parsed: lockfile content object + err: any errors from pasring + + Returns: + A tuple of (importers dict, packages dict, patched_dependencies dict, error string) + """ + if err != None or parsed == None or parsed == {}: + return {}, {}, {}, err + + if not types.is_dict(parsed): + return {}, {}, {}, "lockfile should be a starlark dict" + if "lockfileVersion" not in parsed.keys(): + return {}, {}, {}, "expected lockfileVersion key in lockfile" + + # Lockfile version may be a float such as 5.4 or a string such as '6.0' + lockfile_version = str(parsed["lockfileVersion"]) + lockfile_version = lockfile_version.lstrip("'") + lockfile_version = lockfile_version.rstrip("'") + lockfile_version = lockfile_version.lstrip("\"") + lockfile_version = lockfile_version.rstrip("\"") + lockfile_version = float(lockfile_version) + _assert_lockfile_version(lockfile_version) + + # Fallback to {".": parsed} for non-workspace lockfiles where the deps are at the root. + importers = parsed.get("importers", {".": parsed}) + packages = parsed.get("packages", {}) + patched_dependencies = parsed.get("patchedDependencies", {}) + + if lockfile_version < 6.0: + importers = _convert_v5_importers(importers) + packages = _convert_v5_packages(packages) + elif lockfile_version < 9.0: + importers = _convert_v6_importers(importers) + packages = _convert_v6_packages(packages) + else: # >= 9 + snapshots = parsed.get("snapshots", {}) + importers = _convert_v9_importers(importers) + packages = _convert_v9_packages(packages, snapshots) + + importers = utils.sorted_map(importers) + packages = utils.sorted_map(packages) + + return importers, packages, patched_dependencies, lockfile_version, None + +def _assert_lockfile_version(version, testonly = False): + if type(version) != type(1.0): + fail("version should be passed as a float") + + # Restrict the supported lock file versions to what this code has been tested with: + # 5.4 - pnpm v7.0.0 bumped the lockfile version to 5.4 + # 6.0 - pnpm v8.0.0 bumped the lockfile version to 6.0; this included breaking changes + # 6.1 - pnpm v8.6.0 bumped the lockfile version to 6.1 + # 9.0 - pnpm v9.0.0 bumped the lockfile version to 9.0 + min_lock_version = 5.4 + max_lock_version = 9.0 + msg = None + + if version < min_lock_version: + msg = "npm_translate_lock requires lock_version at least {min}, but found {actual}. Please upgrade to pnpm v7 or greater.".format( + min = min_lock_version, + actual = version, + ) + if version > max_lock_version: + msg = "npm_translate_lock currently supports a maximum lock_version of {max}, but found {actual}. Please file an issue on rules_js".format( + max = max_lock_version, + actual = version, + ) + if msg and not testonly: + fail(msg) + return msg + +pnpm = struct( + assert_lockfile_version = _assert_lockfile_version, + parse_pnpm_lock_json = _parse_pnpm_lock_json, +) + +# Exported only to be tested +pnpm_test = struct( + strip_v5_peer_dep_or_patched_version = _strip_v5_peer_dep_or_patched_version, +) diff --git a/npm/private/test/parse_pnpm_lock_tests.bzl b/npm/private/test/parse_pnpm_lock_tests.bzl index 6aae5e74f..df55582c9 100644 --- a/npm/private/test/parse_pnpm_lock_tests.bzl +++ b/npm/private/test/parse_pnpm_lock_tests.bzl @@ -1,13 +1,13 @@ "Unit tests for pnpm lock file parsing logic" load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") -load("//npm/private:utils.bzl", "utils") +load("//npm/private:pnpm.bzl", "pnpm", "pnpm_test") def _parse_empty_lock_test_impl(ctx): env = unittest.begin(ctx) - parsed_json_a = utils.parse_pnpm_lock_json("") - parsed_json_b = utils.parse_pnpm_lock_json("{}") + parsed_json_a = pnpm.parse_pnpm_lock_json("") + parsed_json_b = pnpm.parse_pnpm_lock_json("{}") expected = ({}, {}, {}, None) asserts.equals(env, expected, parsed_json_a) @@ -50,7 +50,7 @@ expected_packages = { def _parse_lockfile_v5_test_impl(ctx): env = unittest.begin(ctx) - parsed_json = utils.parse_pnpm_lock_json("""\ + parsed_json = pnpm.parse_pnpm_lock_json("""\ { "lockfileVersion": 5.4, "specifiers": { @@ -91,7 +91,7 @@ def _parse_lockfile_v5_test_impl(ctx): def _parse_lockfile_v6_test_impl(ctx): env = unittest.begin(ctx) - parsed_json = utils.parse_pnpm_lock_json("""\ + parsed_json = pnpm.parse_pnpm_lock_json("""\ { "lockfileVersion": "6.0", "dependencies": { @@ -132,7 +132,7 @@ def _parse_lockfile_v6_test_impl(ctx): def _parse_lockfile_v9_test_impl(ctx): env = unittest.begin(ctx) - parsed_json = utils.parse_pnpm_lock_json("""\ + parsed_json = pnpm.parse_pnpm_lock_json("""\ { "lockfileVersion": "9.0", "settings": { @@ -187,16 +187,52 @@ def _parse_lockfile_v9_test_impl(ctx): return unittest.end(env) +# buildifier: disable=function-docstring +def _test_strip_peer_dep_or_patched_version(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + "21.1.0", + pnpm_test.strip_v5_peer_dep_or_patched_version("21.1.0_rollup@2.70.2_x@1.1.1"), + ) + asserts.equals(env, "1.0.0", pnpm_test.strip_v5_peer_dep_or_patched_version("1.0.0_o3deharooos255qt5xdujc3cuq")) + asserts.equals(env, "21.1.0", pnpm_test.strip_v5_peer_dep_or_patched_version("21.1.0")) + return unittest.end(env) + +# buildifier: disable=function-docstring +def _test_version_supported(ctx): + env = unittest.begin(ctx) + + # Unsupported versions + msgs + msg = pnpm.assert_lockfile_version(5.3, testonly = True) + asserts.equals(env, "npm_translate_lock requires lock_version at least 5.4, but found 5.3. Please upgrade to pnpm v7 or greater.", msg) + msg = pnpm.assert_lockfile_version(1.2, testonly = True) + asserts.equals(env, "npm_translate_lock requires lock_version at least 5.4, but found 1.2. Please upgrade to pnpm v7 or greater.", msg) + msg = pnpm.assert_lockfile_version(99.99, testonly = True) + asserts.equals(env, "npm_translate_lock currently supports a maximum lock_version of 9.0, but found 99.99. Please file an issue on rules_js", msg) + + # supported versions + pnpm.assert_lockfile_version(5.4) + pnpm.assert_lockfile_version(6.0) + pnpm.assert_lockfile_version(6.1) + pnpm.assert_lockfile_version(9.0) + + return unittest.end(env) + a_test = unittest.make(_parse_empty_lock_test_impl, attrs = {}) b_test = unittest.make(_parse_lockfile_v5_test_impl, attrs = {}) c_test = unittest.make(_parse_lockfile_v6_test_impl, attrs = {}) d_test = unittest.make(_parse_lockfile_v9_test_impl, attrs = {}) +e_test = unittest.make(_test_version_supported, attrs = {}) +f_test = unittest.make(_test_strip_peer_dep_or_patched_version, attrs = {}) TESTS = [ a_test, b_test, c_test, d_test, + e_test, + f_test, ] def parse_pnpm_lock_tests(name): diff --git a/npm/private/test/utils_tests.bzl b/npm/private/test/utils_tests.bzl index b733108c4..19960ecd0 100644 --- a/npm/private/test/utils_tests.bzl +++ b/npm/private/test/utils_tests.bzl @@ -6,17 +6,6 @@ load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") load("//npm/private:utils.bzl", "utils", "utils_test") # buildifier: disable=function-docstring -def test_strip_peer_dep_or_patched_version(ctx): - env = unittest.begin(ctx) - asserts.equals( - env, - "21.1.0", - utils_test.strip_v5_peer_dep_or_patched_version("21.1.0_rollup@2.70.2_x@1.1.1"), - ) - asserts.equals(env, "1.0.0", utils_test.strip_v5_peer_dep_or_patched_version("1.0.0_o3deharooos255qt5xdujc3cuq")) - asserts.equals(env, "21.1.0", utils_test.strip_v5_peer_dep_or_patched_version("21.1.0")) - return unittest.end(env) - def test_bazel_name(ctx): env = unittest.begin(ctx) asserts.equals( @@ -34,7 +23,7 @@ def test_bazel_name(ctx): # buildifier: disable=function-docstring def test_pnpm_name(ctx): env = unittest.begin(ctx) - asserts.equals(env, "@scope/y@1.1.1", utils.pnpm_name("@scope/y", "1.1.1")) + asserts.equals(env, "@scope/y@1.1.1", utils.package_key("@scope/y", "1.1.1")) asserts.equals(env, "@scope+y@registry+@scope+y@1.1.1", utils.package_store_name("@scope/y", "registry/@scope/y@1.1.1")) asserts.equals(env, "@scope+y@1.1.1", utils.package_store_name("@scope/y", "1.1.1")) return unittest.end(env) @@ -57,26 +46,6 @@ def test_package_store_name(ctx): asserts.equals(env, "@scope+y@2.1.1", utils.package_store_name("@scope/y", "2.1.1")) return unittest.end(env) -# buildifier: disable=function-docstring -def test_version_supported(ctx): - env = unittest.begin(ctx) - - # Unsupported versions + msgs - msg = utils.assert_lockfile_version(5.3, testonly = True) - asserts.equals(env, "npm_translate_lock requires lock_version at least 5.4, but found 5.3. Please upgrade to pnpm v7 or greater.", msg) - msg = utils.assert_lockfile_version(1.2, testonly = True) - asserts.equals(env, "npm_translate_lock requires lock_version at least 5.4, but found 1.2. Please upgrade to pnpm v7 or greater.", msg) - msg = utils.assert_lockfile_version(99.99, testonly = True) - asserts.equals(env, "npm_translate_lock currently supports a maximum lock_version of 9.0, but found 99.99. Please file an issue on rules_js", msg) - - # supported versions - utils.assert_lockfile_version(5.4) - utils.assert_lockfile_version(6.0) - utils.assert_lockfile_version(6.1) - utils.assert_lockfile_version(9.0) - - return unittest.end(env) - # buildifier: disable=function-docstring def test_parse_package_name(ctx): env = unittest.begin(ctx) @@ -156,12 +125,10 @@ def test_npm_registry_download_url(ctx): ) return unittest.end(env) -t0_test = unittest.make(test_strip_peer_dep_or_patched_version) t1_test = unittest.make(test_bazel_name) t2_test = unittest.make(test_pnpm_name) t3_test = unittest.make(test_friendly_name) t4_test = unittest.make(test_package_store_name) -t5_test = unittest.make(test_version_supported) t6_test = unittest.make(test_parse_package_name) t7_test = unittest.make(test_npm_registry_download_url) t8_test = unittest.make(test_npm_registry_url) @@ -170,12 +137,10 @@ t9_test = unittest.make(test_link_version) def utils_tests(name): unittest.suite( name, - t0_test, t1_test, t2_test, t3_test, t4_test, - t5_test, t6_test, t7_test, t8_test, diff --git a/npm/private/transitive_closure.bzl b/npm/private/transitive_closure.bzl index 776c56536..8680df348 100644 --- a/npm/private/transitive_closure.bzl +++ b/npm/private/transitive_closure.bzl @@ -40,7 +40,7 @@ def gather_transitive_closure(packages, package, no_optional, cache = {}): package_key = version[4:] name, version = package_key.rsplit("@", 1) elif version not in packages: - package_key = utils.pnpm_name(name, version) + package_key = utils.package_key(name, version) else: package_key = version transitive_closure[name] = transitive_closure.get(name, []) diff --git a/npm/private/utils.bzl b/npm/private/utils.bzl index b15381220..44ff00f1e 100644 --- a/npm/private/utils.bzl +++ b/npm/private/utils.bzl @@ -4,7 +4,6 @@ load("@aspect_bazel_lib//lib:paths.bzl", "relative_file") load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils") load("@aspect_bazel_lib//lib:utils.bzl", bazel_lib_utils = "utils") load("@bazel_skylib//lib:paths.bzl", "paths") -load("@bazel_skylib//lib:types.bzl", "types") INTERNAL_ERROR_MSG = "ERROR: rules_js internal error, please file an issue: https://github.com/aspect-build/rules_js/issues" DEFAULT_REGISTRY_DOMAIN = "registry.npmjs.org" @@ -43,575 +42,10 @@ def _bazel_name(name, version = None): escaped_version = "%s__%s" % (escaped_version, _sanitize_string(peer_version)) return "%s__%s" % (escaped_name, escaped_version) -def _to_package_key(name, version): - if not version[0].isdigit(): - return version - return "{}@{}".format(name, version) - -def _strip_v5_peer_dep_or_patched_version(version): - "Remove peer dependency or patched syntax from version string" - - # 21.1.0_rollup@2.70.2 becomes 21.1.0 - # 1.0.0_o3deharooos255qt5xdujc3cuq becomes 1.0.0 - index = version.find("_") - if index != -1: - return version[:index] - return version - -def _pnpm_name(name, version): +def _package_key(name, version): "Make a name/version pnpm-style name for a package name and version" return "%s@%s" % (name, version) -# Metadata about a pnpm "project" (importer). -# -# Metadata may come from different locations depending on the lockfile, this struct should -# have data normalized across lockfiles. -def _new_import_info(dependencies, dev_dependencies, optional_dependencies): - return { - "dependencies": dependencies, - "dev_dependencies": dev_dependencies, - "optional_dependencies": optional_dependencies, - } - -# Metadata about a package. -# -# Metadata may come from different locations depending on the lockfile, this struct should -# have data normalized across lockfiles. -def _new_package_info(id, name, dependencies, optional_dependencies, peer_dependencies, dev, has_bin, optional, requires_build, version, friendly_version, resolution): - return { - "id": id, - "name": name, - "dependencies": dependencies, - "optional_dependencies": optional_dependencies, - "peer_dependencies": peer_dependencies, - "dev": dev, - "has_bin": has_bin, - "optional": optional, - "requires_build": requires_build, - "version": version, - "friendly_version": friendly_version, - "resolution": resolution, - } - -def _strip_default_registry(name_version): - # Strip the default registry from the name_version string - if name_version.startswith(DEFAULT_REGISTRY_DOMAIN_SLASH): - return name_version[len(DEFAULT_REGISTRY_DOMAIN_SLASH):] - return name_version - -def _strip_v5_default_registry_to_version(name, version): - # Strip the default registry/name/ from the version string - pre = DEFAULT_REGISTRY_DOMAIN_SLASH + name + "/" - if version.startswith(pre): - return version[len(pre):] - return version - -def _convert_v5_importer_dependency_map(specifiers, deps): - result = {} - for name, version in deps.items(): - specifier = specifiers.get(name) - - if specifier.startswith("npm:"): - # Keep the npm: specifier for aliased dependencies - # convert v5 style aliases ([default_registry]/aliased/version) to npm:aliased@version - alias, version = _strip_default_registry(version).lstrip("/").rsplit("/", 1) - version = "npm:{}@{}".format(alias, version) - else: - # Transition [registry/]name/version[_patch][_peer_data] to a rules_js version format - version = _convert_pnpm_v5_version_peer_dep(_strip_v5_default_registry_to_version(name, version)) - - result[name] = version - return result - -def _convert_v5_importers(importers): - result = {} - for import_path, importer in importers.items(): - specifiers = importer.get("specifiers", {}) - - result[import_path] = _new_import_info( - dependencies = _convert_v5_importer_dependency_map(specifiers, importer.get("dependencies", {})), - dev_dependencies = _convert_v5_importer_dependency_map(specifiers, importer.get("devDependencies", {})), - optional_dependencies = _convert_v5_importer_dependency_map(specifiers, importer.get("optionalDependencies", {})), - ) - return result - -def _convert_pnpm_v5_version_peer_dep(version): - # Covert a pnpm lock file v5 version string of the format - # 1.2.3_@scope+peer@2.0.2_@scope+peer@4.5.6 - # to a version_peer_version that is compatible with rules_js. - - # If there is a suffix to the version - peer_dep_index = version.find("_") - if peer_dep_index != -1: - # if the suffix contains an @version (not just a _patchhash) - peer_dep_index = version.find("@", peer_dep_index) - if peer_dep_index != -1: - peer_dep = version[peer_dep_index:] - version = version[0:peer_dep_index] + _sanitize_string(peer_dep) - - return version - -def _convert_pnpm_v5_package_dependency_version(name, version): - # an alias to an alternate package - if version.startswith("/"): - alias, version = version[1:].rsplit("/", 1) - return "npm:{}@{}".format(alias, version) - - # Removing the default registry+name from the version string - version = _strip_v5_default_registry_to_version(name, version) - - # Convert peer dependency data to rules_js ~v5 format - version = _convert_pnpm_v5_version_peer_dep(version) - - return version - -def _convert_pnpm_v5_package_dependency_map(deps): - result = {} - for name, version in deps.items(): - result[name] = _convert_pnpm_v5_package_dependency_version(name, version) - return result - -def _convert_v5_v6_file_package(package_path, package_snapshot): - if "name" not in package_snapshot: - msg = "expected package {} to have a name field".format(package_path) - fail(msg) - - name = package_snapshot["name"] - version = package_path - if _is_vendored_tarfile(package_snapshot): - if "version" in package_snapshot: - version = package_snapshot["version"] - friendly_version = version - else: - friendly_version = package_snapshot["version"] if "version" in package_snapshot else version - - return name, version, friendly_version - -def _convert_v5_packages(packages): - result = {} - for package_path, package_snapshot in packages.items(): - if "resolution" not in package_snapshot: - msg = "package {} has no resolution field".format(package_path) - fail(msg) - - package_path = _convert_pnpm_v5_version_peer_dep(package_path) - - if package_path.startswith("file:"): - # direct reference to file - name, version, friendly_version = _convert_v5_v6_file_package(package_path, package_snapshot) - elif "name" in package_snapshot and "version" in package_snapshot: - # key/path is complicated enough the real name+version are properties - name = package_snapshot["name"] - version = _strip_v5_default_registry_to_version(name, package_path) - friendly_version = package_snapshot["version"] - elif package_path.startswith("/"): - # a simple /name/version[_peer_info] - name, version = package_path[1:].rsplit("/", 1) - friendly_version = _strip_v5_peer_dep_or_patched_version(version) - else: - msg = "unexpected package path: {} of {}".format(package_path, package_snapshot) - fail(msg) - - package_key = _to_package_key(name, version) - - package_info = _new_package_info( - id = package_snapshot.get("id", None), - name = name, - version = version, - friendly_version = friendly_version, - dependencies = _convert_pnpm_v5_package_dependency_map(package_snapshot.get("dependencies", {})), - peer_dependencies = _convert_pnpm_v5_package_dependency_map(package_snapshot.get("peerDependencies", {})), - optional_dependencies = _convert_pnpm_v5_package_dependency_map(package_snapshot.get("optionalDependencies", {})), - dev = package_snapshot.get("dev", False), - has_bin = package_snapshot.get("hasBin", False), - optional = package_snapshot.get("optional", False), - requires_build = package_snapshot.get("requiresBuild", False), - resolution = package_snapshot.get("resolution"), - ) - - if package_key in result: - msg = "WARNING: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) - - # buildifier: disable=print - print(msg) - - result[package_key] = package_info - return result - -def _convert_pnpm_v6_v9_version_peer_dep(version): - # Covert a pnpm lock file v6 version string of the format - # 1.2.3(@scope/peer@2.0.2)(@scope/peer@4.5.6) - # to a version_peer_version that is compatible with rules_js. - if version[-1] == ")": - # Drop the patch_hash= not present in v5 so (patch_hash=123) -> (123) like v5 - version = version.replace("(patch_hash=", "(") - - # There is a peer dep if the string ends with ")" - peer_dep_index = version.find("(") - peer_dep = version[peer_dep_index:] - if len(peer_dep) > 32: - # Prevent long paths. The pnpm lockfile v6 no longer hashes long sequences of - # peer deps so we must hash here to prevent extremely long file paths that lead to - # "File name too long) build failures. - peer_dep = "_" + _hash(peer_dep) - version = version[0:peer_dep_index] + _sanitize_string(peer_dep) - version = version.rstrip("_") - return version - -def _strip_v6_default_registry_to_version(name, version): - # Strip the default registry/name@ from the version string - pre = DEFAULT_REGISTRY_DOMAIN_SLASH + name + "@" - if version.startswith(pre): - return version[len(pre):] - return version - -def _convert_pnpm_v6_importer_dependency_map(deps): - result = {} - for name, attributes in deps.items(): - specifier = attributes.get("specifier") - version = attributes.get("version") - - if specifier.startswith("npm:"): - # Keep the npm: specifier for aliased dependencies - # convert v6 style aliases ([registry]/aliased@version) to npm:aliased@version - alias, version = _strip_default_registry(version).lstrip("/").rsplit("@", 1) - version = "npm:{}@{}".format(alias, version) - else: - # Transition [registry/]name@version[(peer)(data)] to a rules_js version format - version = _convert_pnpm_v6_v9_version_peer_dep(_strip_v6_default_registry_to_version(name, version)) - - result[name] = version - return result - -def _convert_v6_importers(importers): - # Convert pnpm lockfile v6 importers to a rules_js compatible ~v5 format. - # - # v5 importers: - # specifiers: - # pkg-a: 1.2.3 - # pkg-b: ^4.5.6 - # deps: - # pkg-a: 1.2.3 - # devDeps: - # pkg-b: 4.10.1 - # ... - # - # v6 pushed the 'specifiers' and 'version' into subproperties: - # - # deps: - # pkg-a: - # specifier: 1.2.3 - # version: 1.2.3 - # devDeps: - # pkg-b: - # specifier: ^4.5.6 - # version: 4.10.1 - - result = {} - for import_path, importer in importers.items(): - result[import_path] = _new_import_info( - dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("dependencies", {})), - dev_dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("devDependencies", {})), - optional_dependencies = _convert_pnpm_v6_importer_dependency_map(importer.get("optionalDependencies", {})), - ) - return result - -def _convert_pnpm_v6_package_dependency_version(name, version): - # an alias to an alternate package - if version.startswith("/"): - # Convert peer dependency data to rules_js ~v5 format - version = _convert_pnpm_v6_v9_version_peer_dep(version[1:]) - - return "npm:{}".format(version) - - # Removing the default registry+name from the version string - version = _strip_v6_default_registry_to_version(name, version) - - # Convert peer dependency data to rules_js ~v5 format - version = _convert_pnpm_v6_v9_version_peer_dep(version) - - return version - -def _convert_pnpm_v6_package_dependency_map(deps): - result = {} - for name, version in deps.items(): - result[name] = _convert_pnpm_v6_package_dependency_version(name, version) - return result - -def _convert_v6_packages(packages): - # Convert pnpm lockfile v6 importers to a rules_js compatible ~v5 format. - # - # v6 package metadata mainly changed formatting of metadata such as: - # - # dependency versions with peers: - # v5: 2.0.0_@aspect-test+c@2.0.2 - # v6: 2.0.0(@aspect-test/c@2.0.2) - - result = {} - for package_path, package_snapshot in packages.items(): - if "resolution" not in package_snapshot: - msg = "package {} has no resolution field".format(package_path) - fail(msg) - - package_path = _convert_pnpm_v6_v9_version_peer_dep(package_path) - - if package_path.startswith("file:"): - # direct reference to file - name, version, friendly_version = _convert_v5_v6_file_package(package_path, package_snapshot) - elif "name" in package_snapshot and "version" in package_snapshot: - # key/path is complicated enough the real name+version are properties - name = package_snapshot["name"] - version = _strip_v6_default_registry_to_version(name, package_path) - friendly_version = package_snapshot["version"] - elif package_path.startswith("/"): - # plain /pkg@version(_peer_info) - name, version = package_path[1:].rsplit("@", 1) - friendly_version = _strip_v5_peer_dep_or_patched_version(version) # NOTE: already converted to v5 peer style - else: - msg = "unexpected package path: {} of {}".format(package_path, package_snapshot) - fail(msg) - - package_key = _to_package_key(name, version) - - package_info = _new_package_info( - id = package_snapshot.get("id", None), - name = name, - version = version, - friendly_version = friendly_version, - dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("dependencies", {})), - peer_dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("peerDependencies", {})), - optional_dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("optionalDependencies", {})), - dev = package_snapshot.get("dev", False), - has_bin = package_snapshot.get("hasBin", False), - optional = package_snapshot.get("optional", False), - requires_build = package_snapshot.get("requiresBuild", False), - resolution = package_snapshot.get("resolution"), - ) - - if package_key in result: - msg = "ERROR: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) - - # buildifier: disable=print - print(msg) - - result[package_key] = package_info - - return result - -def _convert_pnpm_v9_package_dependency_version(snapshots, name, version): - # Detect when an alias is just a direct reference to another snapshot - is_alias = version in snapshots - - # Convert peer dependency data to rules_js ~v5 format - version = _convert_pnpm_v6_v9_version_peer_dep(version) - - return "npm:{}".format(version) if is_alias else version - -def _convert_pnpm_v9_package_dependency_map(snapshots, deps): - result = {} - for name, version in deps.items(): - result[name] = _convert_pnpm_v9_package_dependency_version(snapshots, name, version) - return result - -def _convert_pnpm_v9_importer_dependency_map(deps): - result = {} - for name, attributes in deps.items(): - specifier = attributes.get("specifier") - version = attributes.get("version") - - # Transition version[(patch)(peer)(data)] to a rules_js version format - version = _convert_pnpm_v6_v9_version_peer_dep(version) - - if specifier.startswith("npm:"): - # Keep the npm: specifier for aliased dependencies - alias, version = version.rsplit("@", 1) - version = "npm:{}@{}".format(alias, version) - - result[name] = version - return result - -def _convert_v9_importers(importers): - # Convert pnpm lockfile v9 importers to a rules_js compatible ~v5 format. - # Almost identical to v6 but with fewer odd edge cases. - - result = {} - for import_path, importer in importers.items(): - result[import_path] = _new_import_info( - dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("dependencies", {})), - dev_dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("devDependencies", {})), - optional_dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("optionalDependencies", {})), - ) - return result - -def _convert_v9_packages(packages, snapshots): - # Convert pnpm lockfile v9 importers to a rules_js compatible format. - - # v9 split package metadata (v6 "packages" field) into 2: - # - # The 'snapshots' keys contain the resolved dependencies such as each unique combo of deps/peers/versions - # while 'packages' contain the static information about each and every package@version such as hasBin, - # resolution and static dep data. - # - # Note all static registry info such as URLs has moved from the 'importers[x/pkg@version].version' and 'packages[x/pkg@version]' to - # only being present in the actual packages[pkg@version].resolution.* - # - # Example: - # - # packages: - # '@scoped/name@5.0.2' - # hasBin - # resolution (registry-url, integrity etc) - # peerDependencies which *might* be resolved - # - # snapshots: - # pkg@http://a/url - # ... - # - # '@scoped/name@2.0.0(peer@2.0.2)' - # dependencies: - # a-dep: 1.2.3 - # peer: 2.0.2 - # b-dep: 3.2.1(peer-b@4.5.6) - # alias: actual@1.2.3 - # l: file:../path/to/dir - # x: https://a/url/v1.2.3.tar.gz - - result = {} - - # Snapshots contains the packages with the keys (which include peers) to return - for package_key, package_snapshot in snapshots.items(): - peer_meta_index = package_key.find("(") - static_key = package_key[:peer_meta_index] if peer_meta_index > 0 else package_key - if not static_key in packages: - msg = "package {} not found in pnpm 'packages'".format(static_key) - fail(msg) - - package_data = packages[static_key] - - if "resolution" not in package_data: - msg = "package {} has no resolution field".format(static_key) - fail(msg) - - # the raw name + version are the key, not including peerDeps+patch - version_index = static_key.index("@", 1) - name = static_key[:version_index] - package_key = _convert_pnpm_v6_v9_version_peer_dep(package_key) - - # Extract the version including peerDeps+patch from the key - version = package_key[package_key.index("@", 1) + 1:] - - # package_data can have the resolved "version" for things like https:// deps - friendly_version = package_data["version"] if "version" in package_data else static_key[version_index + 1:] - - package_info = _new_package_info( - id = package_data.get("id", None), # TODO: does v9 have "id"? - name = name, - version = version, - friendly_version = friendly_version, - dependencies = _convert_pnpm_v9_package_dependency_map(snapshots, package_snapshot.get("dependencies", {})), - optional_dependencies = _convert_pnpm_v9_package_dependency_map(snapshots, package_snapshot.get("optionalDependencies", {})), - peer_dependencies = package_data.get("peerDependencies", {}), - dev = None, # TODO(pnpm9): must inspect importers.*.devDependencies? - has_bin = package_data.get("hasBin", False), - optional = package_snapshot.get("optional", False), - requires_build = None, # Unknown from lockfile in v9 - resolution = package_data.get("resolution"), - ) - - if package_key in result: - msg = "ERROR: duplicate package: {}\n\t{}\n\t{}".format(package_key, result[package_key], package_info) - fail(msg) - - result[package_key] = package_info - - return result - -def _parse_pnpm_lock_json(content): - """Parse the content of a pnpm-lock.yaml file. - - Args: - content: lockfile content as json - - Returns: - A tuple of (importers dict, packages dict, patched_dependencies dict, error string) - """ - return _parse_pnpm_lock_common(json.decode(content) if content else None, None) - -def _parse_pnpm_lock_common(parsed, err): - """Helper function used by _parse_pnpm_lock_json. - - Args: - parsed: lockfile content object - err: any errors from pasring - - Returns: - A tuple of (importers dict, packages dict, patched_dependencies dict, error string) - """ - if err != None or parsed == None or parsed == {}: - return {}, {}, {}, err - - if not types.is_dict(parsed): - return {}, {}, {}, "lockfile should be a starlark dict" - if "lockfileVersion" not in parsed.keys(): - return {}, {}, {}, "expected lockfileVersion key in lockfile" - - # Lockfile version may be a float such as 5.4 or a string such as '6.0' - lockfile_version = str(parsed["lockfileVersion"]) - lockfile_version = lockfile_version.lstrip("'") - lockfile_version = lockfile_version.rstrip("'") - lockfile_version = lockfile_version.lstrip("\"") - lockfile_version = lockfile_version.rstrip("\"") - lockfile_version = float(lockfile_version) - _assert_lockfile_version(lockfile_version) - - # Fallback to {".": parsed} for non-workspace lockfiles where the deps are at the root. - importers = parsed.get("importers", {".": parsed}) - packages = parsed.get("packages", {}) - patched_dependencies = parsed.get("patchedDependencies", {}) - - if lockfile_version < 6.0: - importers = _convert_v5_importers(importers) - packages = _convert_v5_packages(packages) - elif lockfile_version < 9.0: - importers = _convert_v6_importers(importers) - packages = _convert_v6_packages(packages) - else: # >= 9 - snapshots = parsed.get("snapshots", {}) - importers = _convert_v9_importers(importers) - packages = _convert_v9_packages(packages, snapshots) - - importers = _sorted_map(importers) - packages = _sorted_map(packages) - - return importers, packages, patched_dependencies, lockfile_version, None - -def _assert_lockfile_version(version, testonly = False): - if type(version) != type(1.0): - fail("version should be passed as a float") - - # Restrict the supported lock file versions to what this code has been tested with: - # 5.4 - pnpm v7.0.0 bumped the lockfile version to 5.4 - # 6.0 - pnpm v8.0.0 bumped the lockfile version to 6.0; this included breaking changes - # 6.1 - pnpm v8.6.0 bumped the lockfile version to 6.1 - # 9.0 - pnpm v9.0.0 bumped the lockfile version to 9.0 - min_lock_version = 5.4 - max_lock_version = 9.0 - msg = None - - if version < min_lock_version: - msg = "npm_translate_lock requires lock_version at least {min}, but found {actual}. Please upgrade to pnpm v7 or greater.".format( - min = min_lock_version, - actual = version, - ) - if version > max_lock_version: - msg = "npm_translate_lock currently supports a maximum lock_version of {max}, but found {actual}. Please file an issue on rules_js".format( - max = max_lock_version, - actual = version, - ) - if msg and not testonly: - fail(msg) - return msg - def _friendly_name(name, version): "Make a name@version developer-friendly name for a package name and version" return "%s@%s" % (name, version) @@ -670,7 +104,8 @@ def _npm_registry_download_url(package, version, registries, default_registry): registry.removesuffix("/"), package, package_name_no_scope, - _strip_v5_peer_dep_or_patched_version(version), + # Strip the rules_js peer/patch metadata off the version. See pnpm.bzl + version[:version.find("_")] if version.find("_") != -1 else version, ) def _is_git_repository_url(url): @@ -781,11 +216,6 @@ WARNING: Issue while reading "{npmrc}". Failed to replace env in config: ${{{tok )) return token -def _is_vendored_tarfile(package_snapshot): - if "resolution" in package_snapshot: - return "tarball" in package_snapshot["resolution"] - return False - def _default_external_repository_action_cache(): return DEFAULT_EXTERNAL_REPOSITORY_ACTION_CACHE @@ -820,9 +250,8 @@ def _is_tarball_extension(ext): utils = struct( bazel_name = _bazel_name, sorted_map = _sorted_map, - pnpm_name = _pnpm_name, - assert_lockfile_version = _assert_lockfile_version, - parse_pnpm_lock_json = _parse_pnpm_lock_json, + package_key = _package_key, + sanitize_string = _sanitize_string, friendly_name = _friendly_name, package_store_name = _package_store_name, make_symlink = _make_symlink, @@ -849,5 +278,4 @@ utils = struct( # Exported only to be tested utils_test = struct( parse_package_name = _parse_package_name, - strip_v5_peer_dep_or_patched_version = _strip_v5_peer_dep_or_patched_version, )