Message ID | 20211124144739.2250-13-stefan.herbrechtsmeier-oss@weidmueller.com |
---|---|
State | New |
Headers | show |
Series | Rework npm support | expand |
A description of the changes and how the new code works is missing. How is the SRC_URI formed? How is the appropriate class selected? Alex On Wed, 24 Nov 2021 at 15:48, Stefan Herbrechtsmeier < stefan.herbrechtsmeier-oss@weidmueller.com> wrote: > From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com> > > Signed-off-by: Stefan Herbrechtsmeier < > stefan.herbrechtsmeier@weidmueller.com> > > --- > > scripts/lib/recipetool/create_npm.py | 243 ++++++++++++++++++++++++--- > 1 file changed, 222 insertions(+), 21 deletions(-) > > diff --git a/scripts/lib/recipetool/create_npm.py > b/scripts/lib/recipetool/create_npm.py > index 3394a89970..296b84340e 100644 > --- a/scripts/lib/recipetool/create_npm.py > +++ b/scripts/lib/recipetool/create_npm.py > @@ -39,6 +39,14 @@ class NpmRecipeHandler(RecipeHandler): > name = name.strip("-") > return name > > + @staticmethod > + def _node_recipe_name(name): > + """Generate a OE friendly Node.js recipe name""" > + name = NpmRecipeHandler._npm_name(name) > + if not name.startswith("node-"): > + name = "node-" + name > + return name > + > @staticmethod > def _get_registry(lines): > """Get the registry value from the 'npm://registry' url""" > @@ -54,6 +62,24 @@ class NpmRecipeHandler(RecipeHandler): > > return registry > > + @staticmethod > + def _get_srcdir(lines): > + """Get the source directory value from the url""" > + srcdir = "" > + > + def _handle_srcdir(varname, origvalue, op, newlines): > + nonlocal srcdir > + if origvalue.startswith("${WORKDIR}"): > + srcdir = origvalue[11:] > + else: > + srcdir = "${BP}" > + > + return origvalue, None, 0, True > + > + bb.utils.edit_metadata(lines, ["S"], _handle_srcdir) > + > + return srcdir > + > @staticmethod > def _ensure_npm(): > """Check if the 'npm' command is available in the recipes""" > @@ -116,6 +142,118 @@ class NpmRecipeHandler(RecipeHandler): > > return os.path.join(srctree, "npm-shrinkwrap.json") > > + def _process_shrinkwrap(self, srctree, shrinkwrap, srcdir): > + """ > + Extract package urls from shrinkwrap dependencies > + """ > + > + urls = [] > + > + def _populate_modules(name, params, deptree): > + from bb.fetch2 import URI > + from bb.fetch2.npm import npm_integrity > + from bb.fetch2.npm import npm_localfile > + > + destsubdirs = [os.path.join("node_modules", dep) for dep in > deptree] > + destsuffix = os.path.join(srcdir, *destsubdirs) > + > + dev = params.get("dev", False) > + integrity = params.get("integrity") > + resolved = params.get("resolved") > + version = params.get("version") > + requires = params.get("requires", {}) > + > + # Handle registry sources > + if bb.utils.is_semver(version) and integrity: > + # Skip dependencies without url > + if not resolved: > + return > + > + pkgv = version > + > + uri = URI(resolved) > + uri.params["downloadfilename"] = npm_localfile(name, > version) > + > + checksum_name, checksum_expected = > npm_integrity(integrity) > + uri.params[checksum_name] = checksum_expected > + > + uri.params["subdir"] = destsuffix > + uri.params["striplevel"] = "1" > + > + url = str(uri) > + > + # Handle http tarball sources > + elif version.startswith("http") and integrity: > + checksum_name, checksum_expected = > npm_integrity(integrity) > + > + pkgv = checksum_expected[:13] > + > + uri = URI(version) > + uri.params["downloadfilename"] = npm_localfile(name, pkgv) > + > + uri.params[checksum_name] = checksum_expected > + > + uri.params["destsuffix"] = destsuffix > + uri.params["striplevel"] = "1" > + > + url = str(uri) > + > + # Handle git sources > + elif version.startswith("git"): > + if version.startswith("github:"): > + version = "git+https://github.com/" + > version[len("github:"):] > + regex = re.compile(r""" > + ^ > + git\+ > + (?P<protocol>[a-z]+) > + :// > + (?P<url>[^#]+) > + \# > + (?P<rev>[0-9a-f]+) > + $ > + """, re.VERBOSE) > + > + match = regex.match(version) > + > + if not match: > + raise Exception("Invalid git url: %s - %s" % > (version, url)) > + > + groups = match.groupdict() > + > + pkgv = str(groups["rev"])[:10] > + > + uri = URI("git://" + str(groups["url"])) > + uri.params["destsuffix"] = destsuffix > + uri.params["nobranch"] = "1" > + uri.params["protocol"] = str(groups["protocol"]) > + uri.params["rev"] = str(groups["rev"]) > + > + url = str(uri) > + > + else: > + raise Exception("Unsupported dependency: %s - %s" % > (name, version)) > + > + # Set package version in shrinkwrap for dependency resolution > + params["pkgv"] = pkgv > + > + urls.append(url) > + > + def _foreach_shrinkwrap_dependency(shrinkwrap, callback): > + def _walk_dependencies(deps, deptree): > + for name in deps: > + subtree = [*deptree, name] > + _walk_dependencies(deps[name].get("dependencies", > {}), subtree) > + if deps[name].get("bundled", False): > + continue > + callback(name, deps[name], subtree) > + > + _walk_dependencies(shrinkwrap.get("dependencies", {}), []) > + > + _foreach_shrinkwrap_dependency(shrinkwrap, _populate_modules) > + > + return urls > + > + > def _handle_licenses(self, srctree, shrinkwrap_file, dev): > """Return the extra license files and the list of packages""" > licfiles = [] > @@ -173,7 +311,7 @@ class NpmRecipeHandler(RecipeHandler): > if "name" not in data or "version" not in data: > return False > > - extravalues["PN"] = self._npm_name(data["name"]) > + extravalues["PN"] = self._node_recipe_name(data["name"]) > extravalues["PV"] = data["version"] > > if "description" in data: > @@ -184,6 +322,24 @@ class NpmRecipeHandler(RecipeHandler): > > dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", > "0")), False) > registry = self._get_registry(lines_before) > + srcdir = self._get_srcdir(lines_before) > + # Replace reserved directories > + if srcdir == "package": > + srcdir = "npm" > + def _handle_srcuri(varname, origvalue, op, newlines): > + """Update the version value""" > + values = "%s;subdir=%s;striplevel=1" % (origvalue, srcdir) > + return values, None, 4, False > + > + (_, newlines) = bb.utils.edit_metadata(lines_before, > ["SRC_URI"], _handle_srcuri) > + lines_before[:] = [line.rstrip('\n') for line in newlines] > + > + def _handle_srcdir(varname, origvalue, op, newlines): > + value = "${WORKDIR}/%s" % (srcdir) > + return value, None, 0, True > + > + (_, newlines) = bb.utils.edit_metadata(lines_before, ["S"], > _handle_srcdir) > + lines_before[:] = [line.rstrip('\n') for line in newlines] > > bb.note("Checking if npm is available ...") > # The native npm is used here (and not the host one) to ensure > that the > @@ -223,37 +379,65 @@ class NpmRecipeHandler(RecipeHandler): > if os.path.exists(lock_copy): > bb.utils.movefile(lock_copy, lock_file) > > - # Add the shrinkwrap file as 'extrafiles' > - shrinkwrap_copy = shrinkwrap_file + ".copy" > - bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy) > - extravalues.setdefault("extrafiles", {}) > - extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy > - > - url_local = "npmsw://%s" % shrinkwrap_file > - url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json" > - > - if dev: > - url_local += ";dev=1" > - url_recipe += ";dev=1" > - > - # Add the npmsw url in the SRC_URI of the generated recipe > def _handle_srcuri(varname, origvalue, op, newlines): > - """Update the version value and add the 'npmsw://' url""" > + """Update the version value""" > value = origvalue.replace("version=" + data["version"], > "version=${PV}") > value = value.replace("version=latest", "version=${PV}") > values = [line.strip() for line in > value.strip('\n').splitlines()] > - if "dependencies" in shrinkwrap: > - values.append(url_recipe) > return values, None, 4, False > > (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], > _handle_srcuri) > lines_before[:] = [line.rstrip('\n') for line in newlines] > > + urls = self._process_shrinkwrap(srctree, shrinkwrap, srcdir) > + > + # Add the package urls in the SRC_URI of the generated recipe > + def _handle_srcuri(varname, origvalue, op, newlines): > + """Add the package urls and git SRCREVs""" > + values = None > + # Handle SRCREVs > + if varname == "SRCREV": > + newlines.append('SRCREV_FORMAT = "main"') > + newlines.append('SRCREV_main = "%s"' % origvalue) > + # Handle urls > + elif varname == "SRC_URI": > + values = [line.strip() for line in origvalue.split()] > + values = ["%s;name=main" % (v) if v.startswith(("git", > "http")) else v for v in values] > + values.extend(sorted(urls)) > + # Handle hashes > + else: > + newvarname = varname.replace("[", "[main.") > + newlines.append('%s = "%s"' % (newvarname, origvalue)) > + return values, None, 4, False > + > + (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI", > "SRCREV", "SRC_URI\[\w*\]"], _handle_srcuri) > + lines_before[:] = [line.rstrip('\n') for line in newlines] > + > + > + # Add the package urls in the SRC_URI of the generated recipe > + def _handle_sums(varname, origvalue, op, newlines): > + """Add the package urls and git SRCREVs""" > + values = None > + if varname == "SRC_URI[md5sum]": > + # Add the urls > + values = [line.strip() for line in origvalue.split()] > + values = [v + ";name=main" if v.startswith("git", "http") > else v for v in values] > + values.extend(sorted(urls)) > + else: > + # Add git SRCREVs > + newlines.append('SRCREV_FORMAT = "main"') > + newlines.append('SRCREV_main = "%s"' % origvalue) > + return values, None, 4, False > + > + (_, newlines) = bb.utils.edit_metadata(lines_before, > ["SRC_URI[md5sum]", "SRCREV" ], _handle_sums) > + lines_before[:] = [line.rstrip('\n') for line in newlines] > + > # In order to generate correct licence checksums in the recipe the > - # dependencies have to be fetched again using the npmsw url > + # dependencies have to be fetched agai n using the urls > bb.note("Fetching npm dependencies ...") > bb.utils.remove(os.path.join(srctree, "node_modules"), > recurse=True) > - fetcher = bb.fetch2.Fetch([url_local], d) > + srcurls = [url.replace("=%s/node_modules" % srcdir, > "=node_modules") for url in urls] > + fetcher = bb.fetch2.Fetch(srcurls, d) > fetcher.download() > fetcher.unpack(srctree) > > @@ -289,7 +473,24 @@ class NpmRecipeHandler(RecipeHandler): > (licenses, extravalues["LIC_FILES_CHKSUM"]) = > _guess_odd_license(licfiles) > split_pkg_licenses([*licenses, *guess_license(srctree, d)], > packages, lines_after) > > - classes.append("npm") > + # Add suitable npm class > + if dev: > + scripts = data.get("scripts", {}) > + if scripts.get("build"): > + deps = data.get("devDependencies", {}) > + if deps.get("@angular/cli"): > + classes.append("angular") > + elif deps.get("karma"): > + classes.append("karma") > + elif scripts.get("test"): > + classes.append("npm_test") > + else: > + classes.append("npm_build") > + else: > + classes.append("npm") > + else: > + classes.append("npm") > + > handled.append("buildsystem") > > return True > -- > 2.20.1 > >
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index 3394a89970..296b84340e 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -39,6 +39,14 @@ class NpmRecipeHandler(RecipeHandler): name = name.strip("-") return name + @staticmethod + def _node_recipe_name(name): + """Generate a OE friendly Node.js recipe name""" + name = NpmRecipeHandler._npm_name(name) + if not name.startswith("node-"): + name = "node-" + name + return name + @staticmethod def _get_registry(lines): """Get the registry value from the 'npm://registry' url""" @@ -54,6 +62,24 @@ class NpmRecipeHandler(RecipeHandler): return registry + @staticmethod + def _get_srcdir(lines): + """Get the source directory value from the url""" + srcdir = "" + + def _handle_srcdir(varname, origvalue, op, newlines): + nonlocal srcdir + if origvalue.startswith("${WORKDIR}"): + srcdir = origvalue[11:] + else: + srcdir = "${BP}" + + return origvalue, None, 0, True + + bb.utils.edit_metadata(lines, ["S"], _handle_srcdir) + + return srcdir + @staticmethod def _ensure_npm(): """Check if the 'npm' command is available in the recipes""" @@ -116,6 +142,118 @@ class NpmRecipeHandler(RecipeHandler): return os.path.join(srctree, "npm-shrinkwrap.json") + def _process_shrinkwrap(self, srctree, shrinkwrap, srcdir): + """ + Extract package urls from shrinkwrap dependencies + """ + + urls = [] + + def _populate_modules(name, params, deptree): + from bb.fetch2 import URI + from bb.fetch2.npm import npm_integrity + from bb.fetch2.npm import npm_localfile + + destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] + destsuffix = os.path.join(srcdir, *destsubdirs) + + dev = params.get("dev", False) + integrity = params.get("integrity") + resolved = params.get("resolved") + version = params.get("version") + requires = params.get("requires", {}) + + # Handle registry sources + if bb.utils.is_semver(version) and integrity: + # Skip dependencies without url + if not resolved: + return + + pkgv = version + + uri = URI(resolved) + uri.params["downloadfilename"] = npm_localfile(name, version) + + checksum_name, checksum_expected = npm_integrity(integrity) + uri.params[checksum_name] = checksum_expected + + uri.params["subdir"] = destsuffix + uri.params["striplevel"] = "1" + + url = str(uri) + + # Handle http tarball sources + elif version.startswith("http") and integrity: + checksum_name, checksum_expected = npm_integrity(integrity) + + pkgv = checksum_expected[:13] + + uri = URI(version) + uri.params["downloadfilename"] = npm_localfile(name, pkgv) + + uri.params[checksum_name] = checksum_expected + + uri.params["destsuffix"] = destsuffix + uri.params["striplevel"] = "1" + + url = str(uri) + + # Handle git sources + elif version.startswith("git"): + if version.startswith("github:"): + version = "git+https://github.com/" + version[len("github:"):] + regex = re.compile(r""" + ^ + git\+ + (?P<protocol>[a-z]+) + :// + (?P<url>[^#]+) + \# + (?P<rev>[0-9a-f]+) + $ + """, re.VERBOSE) + + match = regex.match(version) + + if not match: + raise Exception("Invalid git url: %s - %s" % (version, url)) + + groups = match.groupdict() + + pkgv = str(groups["rev"])[:10] + + uri = URI("git://" + str(groups["url"])) + uri.params["destsuffix"] = destsuffix + uri.params["nobranch"] = "1" + uri.params["protocol"] = str(groups["protocol"]) + uri.params["rev"] = str(groups["rev"]) + + url = str(uri) + + else: + raise Exception("Unsupported dependency: %s - %s" % (name, version)) + + # Set package version in shrinkwrap for dependency resolution + params["pkgv"] = pkgv + + urls.append(url) + + def _foreach_shrinkwrap_dependency(shrinkwrap, callback): + def _walk_dependencies(deps, deptree): + for name in deps: + subtree = [*deptree, name] + _walk_dependencies(deps[name].get("dependencies", {}), subtree) + if deps[name].get("bundled", False): + continue + callback(name, deps[name], subtree) + + _walk_dependencies(shrinkwrap.get("dependencies", {}), []) + + _foreach_shrinkwrap_dependency(shrinkwrap, _populate_modules) + + return urls + + def _handle_licenses(self, srctree, shrinkwrap_file, dev): """Return the extra license files and the list of packages""" licfiles = [] @@ -173,7 +311,7 @@ class NpmRecipeHandler(RecipeHandler): if "name" not in data or "version" not in data: return False - extravalues["PN"] = self._npm_name(data["name"]) + extravalues["PN"] = self._node_recipe_name(data["name"]) extravalues["PV"] = data["version"] if "description" in data: @@ -184,6 +322,24 @@ class NpmRecipeHandler(RecipeHandler): dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False) registry = self._get_registry(lines_before) + srcdir = self._get_srcdir(lines_before) + # Replace reserved directories + if srcdir == "package": + srcdir = "npm" + def _handle_srcuri(varname, origvalue, op, newlines): + """Update the version value""" + values = "%s;subdir=%s;striplevel=1" % (origvalue, srcdir) + return values, None, 4, False + + (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri) + lines_before[:] = [line.rstrip('\n') for line in newlines] + + def _handle_srcdir(varname, origvalue, op, newlines): + value = "${WORKDIR}/%s" % (srcdir) + return value, None, 0, True + + (_, newlines) = bb.utils.edit_metadata(lines_before, ["S"], _handle_srcdir) + lines_before[:] = [line.rstrip('\n') for line in newlines] bb.note("Checking if npm is available ...") # The native npm is used here (and not the host one) to ensure that the @@ -223,37 +379,65 @@ class NpmRecipeHandler(RecipeHandler): if os.path.exists(lock_copy): bb.utils.movefile(lock_copy, lock_file) - # Add the shrinkwrap file as 'extrafiles' - shrinkwrap_copy = shrinkwrap_file + ".copy" - bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy) - extravalues.setdefault("extrafiles", {}) - extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy - - url_local = "npmsw://%s" % shrinkwrap_file - url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json" - - if dev: - url_local += ";dev=1" - url_recipe += ";dev=1" - - # Add the npmsw url in the SRC_URI of the generated recipe def _handle_srcuri(varname, origvalue, op, newlines): - """Update the version value and add the 'npmsw://' url""" + """Update the version value""" value = origvalue.replace("version=" + data["version"], "version=${PV}") value = value.replace("version=latest", "version=${PV}") values = [line.strip() for line in value.strip('\n').splitlines()] - if "dependencies" in shrinkwrap: - values.append(url_recipe) return values, None, 4, False (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri) lines_before[:] = [line.rstrip('\n') for line in newlines] + urls = self._process_shrinkwrap(srctree, shrinkwrap, srcdir) + + # Add the package urls in the SRC_URI of the generated recipe + def _handle_srcuri(varname, origvalue, op, newlines): + """Add the package urls and git SRCREVs""" + values = None + # Handle SRCREVs + if varname == "SRCREV": + newlines.append('SRCREV_FORMAT = "main"') + newlines.append('SRCREV_main = "%s"' % origvalue) + # Handle urls + elif varname == "SRC_URI": + values = [line.strip() for line in origvalue.split()] + values = ["%s;name=main" % (v) if v.startswith(("git", "http")) else v for v in values] + values.extend(sorted(urls)) + # Handle hashes + else: + newvarname = varname.replace("[", "[main.") + newlines.append('%s = "%s"' % (newvarname, origvalue)) + return values, None, 4, False + + (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI", "SRCREV", "SRC_URI\[\w*\]"], _handle_srcuri) + lines_before[:] = [line.rstrip('\n') for line in newlines] + + + # Add the package urls in the SRC_URI of the generated recipe + def _handle_sums(varname, origvalue, op, newlines): + """Add the package urls and git SRCREVs""" + values = None + if varname == "SRC_URI[md5sum]": + # Add the urls + values = [line.strip() for line in origvalue.split()] + values = [v + ";name=main" if v.startswith("git", "http") else v for v in values] + values.extend(sorted(urls)) + else: + # Add git SRCREVs + newlines.append('SRCREV_FORMAT = "main"') + newlines.append('SRCREV_main = "%s"' % origvalue) + return values, None, 4, False + + (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI[md5sum]", "SRCREV" ], _handle_sums) + lines_before[:] = [line.rstrip('\n') for line in newlines] + # In order to generate correct licence checksums in the recipe the - # dependencies have to be fetched again using the npmsw url + # dependencies have to be fetched agai n using the urls bb.note("Fetching npm dependencies ...") bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True) - fetcher = bb.fetch2.Fetch([url_local], d) + srcurls = [url.replace("=%s/node_modules" % srcdir, "=node_modules") for url in urls] + fetcher = bb.fetch2.Fetch(srcurls, d) fetcher.download() fetcher.unpack(srctree) @@ -289,7 +473,24 @@ class NpmRecipeHandler(RecipeHandler): (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles) split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after) - classes.append("npm") + # Add suitable npm class + if dev: + scripts = data.get("scripts", {}) + if scripts.get("build"): + deps = data.get("devDependencies", {}) + if deps.get("@angular/cli"): + classes.append("angular") + elif deps.get("karma"): + classes.append("karma") + elif scripts.get("test"): + classes.append("npm_test") + else: + classes.append("npm_build") + else: + classes.append("npm") + else: + classes.append("npm") + handled.append("buildsystem") return True