[RFC,12/15] recipetool: npm: Add dependencies to SRC_URI and auto select classes

From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>

Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>

---

 scripts/lib/recipetool/create_npm.py | 243 ++++++++++++++++++++++++---
 1 file changed, 222 insertions(+), 21 deletions(-)

A description of the changes and how the new code works is missing. How is
the SRC_URI formed? How is the appropriate class selected?

Alex

On Wed, 24 Nov 2021 at 15:48, Stefan Herbrechtsmeier <
stefan.herbrechtsmeier-oss@weidmueller.com> wrote:

> From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
>
> Signed-off-by: Stefan Herbrechtsmeier <
> stefan.herbrechtsmeier@weidmueller.com>
>
> ---
>
>  scripts/lib/recipetool/create_npm.py | 243 ++++++++++++++++++++++++---
>  1 file changed, 222 insertions(+), 21 deletions(-)
>
> diff --git a/scripts/lib/recipetool/create_npm.py
> b/scripts/lib/recipetool/create_npm.py
> index 3394a89970..296b84340e 100644
> --- a/scripts/lib/recipetool/create_npm.py
> +++ b/scripts/lib/recipetool/create_npm.py
> @@ -39,6 +39,14 @@ class NpmRecipeHandler(RecipeHandler):
>          name = name.strip("-")
>          return name
>
> +    @staticmethod
> +    def _node_recipe_name(name):
> +        """Generate a OE friendly Node.js recipe name"""
> +        name = NpmRecipeHandler._npm_name(name)
> +        if not name.startswith("node-"):
> +            name = "node-" + name
> +        return name
> +
>      @staticmethod
>      def _get_registry(lines):
>          """Get the registry value from the 'npm://registry' url"""
> @@ -54,6 +62,24 @@ class NpmRecipeHandler(RecipeHandler):
>
>          return registry
>
> +    @staticmethod
> +    def _get_srcdir(lines):
> +        """Get the source directory value from the url"""
> +        srcdir = ""
> +
> +        def _handle_srcdir(varname, origvalue, op, newlines):
> +            nonlocal srcdir
> +            if origvalue.startswith("${WORKDIR}"):
> +                srcdir = origvalue[11:]
> +            else:
> +                srcdir = "${BP}"
> +
> +            return origvalue, None, 0, True
> +
> +        bb.utils.edit_metadata(lines, ["S"], _handle_srcdir)
> +
> +        return srcdir
> +
>      @staticmethod
>      def _ensure_npm():
>          """Check if the 'npm' command is available in the recipes"""
> @@ -116,6 +142,118 @@ class NpmRecipeHandler(RecipeHandler):
>
>          return os.path.join(srctree, "npm-shrinkwrap.json")
>
> +    def _process_shrinkwrap(self, srctree, shrinkwrap, srcdir):
> +        """
> +            Extract package urls from shrinkwrap dependencies
> +        """
> +
> +        urls = []
> +
> +        def _populate_modules(name, params, deptree):
> +            from bb.fetch2 import URI
> +            from bb.fetch2.npm import npm_integrity
> +            from bb.fetch2.npm import npm_localfile
> +
> +            destsubdirs = [os.path.join("node_modules", dep) for dep in
> deptree]
> +            destsuffix = os.path.join(srcdir, *destsubdirs)
> +
> +            dev = params.get("dev", False)
> +            integrity = params.get("integrity")
> +            resolved = params.get("resolved")
> +            version = params.get("version")
> +            requires = params.get("requires", {})
> +
> +            # Handle registry sources
> +            if bb.utils.is_semver(version) and integrity:
> +                # Skip dependencies without url
> +                if not resolved:
> +                    return
> +
> +                pkgv = version
> +
> +                uri = URI(resolved)
> +                uri.params["downloadfilename"] = npm_localfile(name,
> version)
> +
> +                checksum_name, checksum_expected =
> npm_integrity(integrity)
> +                uri.params[checksum_name] = checksum_expected
> +
> +                uri.params["subdir"] = destsuffix
> +                uri.params["striplevel"] = "1"
> +
> +                url = str(uri)
> +
> +            # Handle http tarball sources
> +            elif version.startswith("http") and integrity:
> +                checksum_name, checksum_expected =
> npm_integrity(integrity)
> +
> +                pkgv = checksum_expected[:13]
> +
> +                uri = URI(version)
> +                uri.params["downloadfilename"] = npm_localfile(name, pkgv)
> +
> +                uri.params[checksum_name] = checksum_expected
> +
> +                uri.params["destsuffix"] = destsuffix
> +                uri.params["striplevel"] = "1"
> +
> +                url = str(uri)
> +
> +            # Handle git sources
> +            elif version.startswith("git"):
> +                if version.startswith("github:"):
> +                    version = "git+https://github.com/" +
> version[len("github:"):]
> +                regex = re.compile(r"""
> +                    ^
> +                    git\+
> +                    (?P<protocol>[a-z]+)
> +                    ://
> +                    (?P<url>[^#]+)
> +                    \#
> +                    (?P<rev>[0-9a-f]+)
> +                    $
> +                    """, re.VERBOSE)
> +
> +                match = regex.match(version)
> +
> +                if not match:
> +                    raise Exception("Invalid git url: %s - %s" %
> (version, url))
> +
> +                groups = match.groupdict()
> +
> +                pkgv = str(groups["rev"])[:10]
> +
> +                uri = URI("git://" + str(groups["url"]))
> +                uri.params["destsuffix"] = destsuffix
> +                uri.params["nobranch"] = "1"
> +                uri.params["protocol"] = str(groups["protocol"])
> +                uri.params["rev"] = str(groups["rev"])
> +
> +                url = str(uri)
> +
> +            else:
> +                raise Exception("Unsupported dependency: %s - %s" %
> (name, version))
> +
> +            # Set package version in shrinkwrap for dependency resolution
> +            params["pkgv"] = pkgv
> +
> +            urls.append(url)
> +
> +        def _foreach_shrinkwrap_dependency(shrinkwrap, callback):
> +            def _walk_dependencies(deps, deptree):
> +                for name in deps:
> +                    subtree = [*deptree, name]
> +                    _walk_dependencies(deps[name].get("dependencies",
> {}), subtree)
> +                    if deps[name].get("bundled", False):
> +                        continue
> +                    callback(name, deps[name], subtree)
> +
> +            _walk_dependencies(shrinkwrap.get("dependencies", {}), [])
> +
> +        _foreach_shrinkwrap_dependency(shrinkwrap, _populate_modules)
> +
> +        return urls
> +
> +
>      def _handle_licenses(self, srctree, shrinkwrap_file, dev):
>          """Return the extra license files and the list of packages"""
>          licfiles = []
> @@ -173,7 +311,7 @@ class NpmRecipeHandler(RecipeHandler):
>          if "name" not in data or "version" not in data:
>              return False
>
> -        extravalues["PN"] = self._npm_name(data["name"])
> +        extravalues["PN"] = self._node_recipe_name(data["name"])
>          extravalues["PV"] = data["version"]
>
>          if "description" in data:
> @@ -184,6 +322,24 @@ class NpmRecipeHandler(RecipeHandler):
>
>          dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV",
> "0")), False)
>          registry = self._get_registry(lines_before)
> +        srcdir = self._get_srcdir(lines_before)
> +        # Replace reserved directories
> +        if srcdir == "package":
> +            srcdir = "npm"
> +            def _handle_srcuri(varname, origvalue, op, newlines):
> +                """Update the version value"""
> +                values = "%s;subdir=%s;striplevel=1" % (origvalue, srcdir)
> +                return values, None, 4, False
> +
> +            (_, newlines) = bb.utils.edit_metadata(lines_before,
> ["SRC_URI"], _handle_srcuri)
> +            lines_before[:] = [line.rstrip('\n') for line in newlines]
> +
> +            def _handle_srcdir(varname, origvalue, op, newlines):
> +                value = "${WORKDIR}/%s" % (srcdir)
> +                return value, None, 0, True
> +
> +            (_, newlines) = bb.utils.edit_metadata(lines_before, ["S"],
> _handle_srcdir)
> +            lines_before[:] = [line.rstrip('\n') for line in newlines]
>
>          bb.note("Checking if npm is available ...")
>          # The native npm is used here (and not the host one) to ensure
> that the
> @@ -223,37 +379,65 @@ class NpmRecipeHandler(RecipeHandler):
>          if os.path.exists(lock_copy):
>              bb.utils.movefile(lock_copy, lock_file)
>
> -        # Add the shrinkwrap file as 'extrafiles'
> -        shrinkwrap_copy = shrinkwrap_file + ".copy"
> -        bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
> -        extravalues.setdefault("extrafiles", {})
> -        extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
> -
> -        url_local = "npmsw://%s" % shrinkwrap_file
> -        url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
> -
> -        if dev:
> -            url_local += ";dev=1"
> -            url_recipe += ";dev=1"
> -
> -        # Add the npmsw url in the SRC_URI of the generated recipe
>          def _handle_srcuri(varname, origvalue, op, newlines):
> -            """Update the version value and add the 'npmsw://' url"""
> +            """Update the version value"""
>              value = origvalue.replace("version=" + data["version"],
> "version=${PV}")
>              value = value.replace("version=latest", "version=${PV}")
>              values = [line.strip() for line in
> value.strip('\n').splitlines()]
> -            if "dependencies" in shrinkwrap:
> -                values.append(url_recipe)
>              return values, None, 4, False
>
>          (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"],
> _handle_srcuri)
>          lines_before[:] = [line.rstrip('\n') for line in newlines]
>
> +        urls = self._process_shrinkwrap(srctree, shrinkwrap, srcdir)
> +
> +        # Add the package urls in the SRC_URI of the generated recipe
> +        def _handle_srcuri(varname, origvalue, op, newlines):
> +            """Add the package urls and git SRCREVs"""
> +            values = None
> +            # Handle SRCREVs
> +            if varname == "SRCREV":
> +                newlines.append('SRCREV_FORMAT = "main"')
> +                newlines.append('SRCREV_main = "%s"' % origvalue)
> +            # Handle urls
> +            elif varname == "SRC_URI":
> +                values = [line.strip() for line in origvalue.split()]
> +                values = ["%s;name=main" % (v) if v.startswith(("git",
> "http")) else v for v in values]
> +                values.extend(sorted(urls))
> +            # Handle hashes
> +            else:
> +                newvarname = varname.replace("[", "[main.")
> +                newlines.append('%s = "%s"' % (newvarname, origvalue))
> +            return values, None, 4, False
> +
> +        (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI",
> "SRCREV", "SRC_URI\[\w*\]"], _handle_srcuri)
> +        lines_before[:] = [line.rstrip('\n') for line in newlines]
> +
> +
> +        # Add the package urls in the SRC_URI of the generated recipe
> +        def _handle_sums(varname, origvalue, op, newlines):
> +            """Add the package urls and git SRCREVs"""
> +            values = None
> +            if varname == "SRC_URI[md5sum]":
> +                # Add the urls
> +                values = [line.strip() for line in origvalue.split()]
> +                values = [v + ";name=main" if v.startswith("git", "http")
> else v for v in values]
> +                values.extend(sorted(urls))
> +            else:
> +                # Add git SRCREVs
> +                newlines.append('SRCREV_FORMAT = "main"')
> +                newlines.append('SRCREV_main = "%s"' % origvalue)
> +            return values, None, 4, False
> +
> +        (_, newlines) = bb.utils.edit_metadata(lines_before,
> ["SRC_URI[md5sum]", "SRCREV" ], _handle_sums)
> +        lines_before[:] = [line.rstrip('\n') for line in newlines]
> +
>          # In order to generate correct licence checksums in the recipe the
> -        # dependencies have to be fetched again using the npmsw url
> +        # dependencies have to be fetched agai  n using the urls
>          bb.note("Fetching npm dependencies ...")
>          bb.utils.remove(os.path.join(srctree, "node_modules"),
> recurse=True)
> -        fetcher = bb.fetch2.Fetch([url_local], d)
> +        srcurls = [url.replace("=%s/node_modules" % srcdir,
> "=node_modules") for url in urls]
> +        fetcher = bb.fetch2.Fetch(srcurls, d)
>          fetcher.download()
>          fetcher.unpack(srctree)
>
> @@ -289,7 +473,24 @@ class NpmRecipeHandler(RecipeHandler):
>          (licenses, extravalues["LIC_FILES_CHKSUM"]) =
> _guess_odd_license(licfiles)
>          split_pkg_licenses([*licenses, *guess_license(srctree, d)],
> packages, lines_after)
>
> -        classes.append("npm")
> +        # Add suitable npm class
> +        if dev:
> +            scripts = data.get("scripts", {})
> +            if scripts.get("build"):
> +                deps = data.get("devDependencies", {})
> +                if deps.get("@angular/cli"):
> +                    classes.append("angular")
> +                elif deps.get("karma"):
> +                    classes.append("karma")
> +                elif scripts.get("test"):
> +                    classes.append("npm_test")
> +                else:
> +                    classes.append("npm_build")
> +            else:
> +                classes.append("npm")
> +        else:
> +            classes.append("npm")
> +
>          handled.append("buildsystem")
>
>          return True
> --
> 2.20.1
>
>