new file mode 100644
@@ -0,0 +1,99 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+"""Explicit SRC_URI entries for the dependencies of an npm shrinkwrap file."""
+
+import os
+import re
+
+import bb.fetch
+import bb.utils
+from bb.fetch.npm import npm_integrity, npm_localfile
+from bb.fetch.npmsw import foreach_dependencies
+
+GIT_RESOLVED_RE = re.compile(r"^git\+(?P<protocol>[a-z]+)://(?P<url>[^#]+)#(?P<rev>[0-9a-f]+)$")
+
+
+def shrinkwrap_src_uri(shrinkwrap, destdir, dev=False):
+ """Return one SRC_URI entry for each dependency of a shrinkwrap file.
+
+ shrinkwrap is the content of npm-shrinkwrap.json (lockfileVersion 2 or
+ later). destdir is the directory of the main package, relative to
+ UNPACKDIR. Each dependency unpacks to its location in destdir (for example
+ destdir/node_modules/foo), where npm.bbclass expects it. dev also adds the
+ devDependencies. The entries are sorted by location.
+
+ A package from a registry gets an npm:// entry, so that the fetcher knows
+ its name and version. Other tarballs get a plain URL and git dependencies
+ a git:// URL. Each entry has the checksum or revision of the shrinkwrap
+ file.
+ """
+ entries = {}
+
+ def _add(name, data, location):
+ package = data.get("name", name)
+ version = data.get("version")
+ resolved = data.get("resolved")
+ integrity = data.get("integrity")
+ destsuffix = os.path.join(destdir, location)
+
+ if not resolved:
+ # npm leaves out "resolved" for some duplicate packages. npm.bbclass
+ # does not need these, as npmsw already skipped them.
+ if version and integrity:
+ return
+ raise ValueError("%s: no 'resolved' field" % location)
+
+ if data.get("link") or resolved.startswith("file:"):
+ raise ValueError("%s: local dependencies are not supported" % location)
+
+ match = GIT_RESOLVED_RE.match(resolved)
+ if match:
+ entries[location] = "git://%s;protocol=%s;rev=%s;nobranch=1;destsuffix=%s" % (
+ match["url"], match["protocol"], match["rev"], destsuffix)
+ return
+
+ if not integrity or not resolved.startswith(("https://", "http://")):
+ raise ValueError("%s: unsupported dependency %s" % (location, resolved))
+
+ checksum_name, checksum = npm_integrity(integrity)
+ basename = package.split("/")[-1]
+ suffix = "/%s/-/%s-%s.tgz" % (package, basename, version)
+ if resolved.startswith("https://") and resolved.endswith(suffix) and bb.utils.is_semver(version):
+ registry = resolved[len("https://"):-len(suffix)]
+ entries[location] = "npm://%s/;package=%s;version=%s;%s=%s;destsuffix=%s" % (
+ registry, package, version, checksum_name, checksum, destsuffix)
+ else:
+ entries[location] = "%s;downloadfilename=%s;%s=%s;subdir=%s;striplevel=1" % (
+ resolved, npm_localfile(os.path.basename(resolved)), checksum_name, checksum, destsuffix)
+
+ foreach_dependencies(shrinkwrap, _add, dev)
+ return [entries[location] for location in sorted(entries)]
+
+
+def src_uri_differences(expected, src_uri):
+ """Compare the expected entries with SRC_URI.
+
+ Returns (missing, extra): the expected entries that SRC_URI does not have,
+ and the SRC_URI entries that unpack into a node_modules directory but are
+ not expected. Entries are compared after parsing, so the order of the URL
+ parameters does not matter.
+ """
+ def _key(url):
+ scheme, host, path, user, password, parm = bb.fetch.decodeurl(url)
+ return (scheme, host, path, user, password, tuple(sorted(parm.items())))
+
+ def _unpacks_to_node_modules(url):
+ parm = bb.fetch.decodeurl(url)[5]
+ dest = parm.get("destsuffix") or parm.get("subdir") or ""
+ return "node_modules/" in dest + "/"
+
+ expected_keys = {_key(url): url for url in expected}
+ actual_keys = {_key(url): url for url in src_uri}
+ missing = [url for key, url in expected_keys.items() if key not in actual_keys]
+ extra = [url for key, url in actual_keys.items()
+ if key not in expected_keys and _unpacks_to_node_modules(url)]
+ return missing, extra
new file mode 100644
@@ -0,0 +1,121 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+import base64
+from unittest.case import TestCase
+
+import oe.npm
+
+CHECKSUM = bytes(range(64)).hex()
+INTEGRITY = "sha512-" + base64.b64encode(bytes(range(64))).decode()
+
+
+def shrinkwrap(packages):
+ """Return a shrinkwrap file with the given dependencies"""
+ return {"lockfileVersion": 3, "packages": {"": {"name": "main"}, **packages}}
+
+
+def registry(package, version, host="registry.npmjs.org"):
+ """Return a dependency from a registry"""
+ basename = package.split("/")[-1]
+ return {
+ "version": version,
+ "resolved": "https://%s/%s/-/%s-%s.tgz" % (host, package, basename, version),
+ "integrity": INTEGRITY,
+ }
+
+
+class TestShrinkwrapSrcUri(TestCase):
+ def test_registry(self):
+ entries = oe.npm.shrinkwrap_src_uri(shrinkwrap({
+ "node_modules/send": registry("send", "0.19.0"),
+ "node_modules/send/node_modules/ms": registry("ms", "2.1.3"),
+ "node_modules/@scope/pkg": registry("@scope/pkg", "1.0.0", "npm.example.com/path"),
+ }), "npm")
+ self.assertEqual(entries, [
+ "npm://npm.example.com/path/;package=@scope/pkg;version=1.0.0;sha512sum=%s;destsuffix=npm/node_modules/@scope/pkg" % CHECKSUM,
+ "npm://registry.npmjs.org/;package=send;version=0.19.0;sha512sum=%s;destsuffix=npm/node_modules/send" % CHECKSUM,
+ "npm://registry.npmjs.org/;package=ms;version=2.1.3;sha512sum=%s;destsuffix=npm/node_modules/send/node_modules/ms" % CHECKSUM,
+ ])
+
+ def test_alias(self):
+ # An alias installs a package under another name: the "name" field
+ # gives the real package
+ dep = registry("real", "1.0.0")
+ dep["name"] = "real"
+ entries = oe.npm.shrinkwrap_src_uri(shrinkwrap({"node_modules/alias": dep}), "npm")
+ self.assertEqual(entries, [
+ "npm://registry.npmjs.org/;package=real;version=1.0.0;sha512sum=%s;destsuffix=npm/node_modules/alias" % CHECKSUM,
+ ])
+
+ def test_tarball(self):
+ entries = oe.npm.shrinkwrap_src_uri(shrinkwrap({
+ "node_modules/foo": {
+ "version": "1.0.0",
+ "resolved": "https://example.com/downloads/foo.tgz",
+ "integrity": INTEGRITY,
+ },
+ }), "npm")
+ self.assertEqual(entries, [
+ "https://example.com/downloads/foo.tgz;downloadfilename=npm2/foo.tgz;sha512sum=%s;subdir=npm/node_modules/foo;striplevel=1" % CHECKSUM,
+ ])
+
+ def test_git(self):
+ entries = oe.npm.shrinkwrap_src_uri(shrinkwrap({
+ "node_modules/foo": {
+ "version": "1.0.0",
+ "resolved": "git+ssh://git@github.com/example/foo.git#0123456789abcdef0123456789abcdef01234567",
+ },
+ }), "npm")
+ self.assertEqual(entries, [
+ "git://git@github.com/example/foo.git;protocol=ssh;rev=0123456789abcdef0123456789abcdef01234567;nobranch=1;destsuffix=npm/node_modules/foo",
+ ])
+
+ def test_skipped(self):
+ dev = registry("dev", "1.0.0")
+ dev["dev"] = True
+ bundled = registry("bundled", "1.0.0")
+ bundled["inBundle"] = True
+ no_resolved = registry("dup", "1.0.0")
+ del no_resolved["resolved"]
+ packages = {
+ "node_modules/dev": dev,
+ "node_modules/bundled": bundled,
+ "node_modules/dup": no_resolved,
+ }
+ self.assertEqual(oe.npm.shrinkwrap_src_uri(shrinkwrap(packages), "npm"), [])
+ self.assertEqual(len(oe.npm.shrinkwrap_src_uri(shrinkwrap(packages), "npm", dev=True)), 1)
+
+ def test_unsupported(self):
+ for dep in ({"version": "1.0.0", "resolved": "file:../foo"},
+ {"link": True, "resolved": "../foo"},
+ {"version": "1.0.0"}):
+ with self.assertRaises(ValueError):
+ oe.npm.shrinkwrap_src_uri(shrinkwrap({"node_modules/foo": dep}), "npm")
+
+
+class TestSrcUriDifferences(TestCase):
+ EXPECTED = [
+ "npm://registry.npmjs.org/;package=foo;version=1.0.0;sha512sum=%s;destsuffix=npm/node_modules/foo" % CHECKSUM,
+ "npm://registry.npmjs.org/;package=bar;version=2.0.0;sha512sum=%s;destsuffix=npm/node_modules/bar" % CHECKSUM,
+ ]
+
+ def test_match(self):
+ # The order of the entries and of the URL parameters does not matter.
+ # Entries that do not unpack into node_modules are not extra.
+ src_uri = [
+ "npm://registry.npmjs.org/;package=main;version=1.0.0;sha512sum=%s" % CHECKSUM,
+ "file://npm-shrinkwrap.json;subdir=npm",
+ self.EXPECTED[1],
+ "npm://registry.npmjs.org/;destsuffix=npm/node_modules/foo;sha512sum=%s;version=1.0.0;package=foo" % CHECKSUM,
+ ]
+ self.assertEqual(oe.npm.src_uri_differences(self.EXPECTED, src_uri), ([], []))
+
+ def test_missing_and_extra(self):
+ old_bar = "npm://registry.npmjs.org/;package=bar;version=1.0.0;sha512sum=%s;destsuffix=npm/node_modules/bar" % CHECKSUM
+ missing, extra = oe.npm.src_uri_differences(self.EXPECTED, [self.EXPECTED[0], old_bar])
+ self.assertEqual(missing, [self.EXPECTED[1]])
+ self.assertEqual(extra, [old_bar])