| Message ID | 20230604013919.13904-1-m.belouarga@technologyandstrategy.com |
|---|---|
| State | New |
| Headers | show |
| Series | [master,v2,1/3] bitbake: fetch2: npmsw: Add support for the new format of the shrinkwrap file | expand |
Hello,
this series fails on the autobuilders:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/5298/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/127/builds/1548/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/5288/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/5323/steps/11/logs/stdio
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/5244/steps/11/logs/stdio
ERROR: test_npmsw_no_network_no_tarball (bb.tests.fetch.NPMTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/pokybuild/yocto-worker/oe-selftest-centos/build/bitbake/lib/bb/tests/fetch.py", line 2850, in test_npmsw_no_network_no_tarball
fetcher.download()
File "/home/pokybuild/yocto-worker/oe-selftest-centos/build/bitbake/lib/bb/fetch2/__init__.py", line 1738, in download
if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
File "/home/pokybuild/yocto-worker/oe-selftest-centos/build/bitbake/lib/bb/fetch2/npmsw.py", line 224, in verify_donestamp
return all(self._foreach_proxy_method(ud, _handle))
File "/home/pokybuild/yocto-worker/oe-selftest-centos/build/bitbake/lib/bb/fetch2/npmsw.py", line 211, in _foreach_proxy_method
for proxy_url in ud.proxy.urls:
AttributeError: 'FetchData' object has no attribute 'proxy'
On 04/06/2023 03:39:17+0200, belouargamohamed@gmail.com wrote:
> From: BELOUARGA Mohamed <m.belouarga@technologyandstrategy.com>
>
> Npm is a package manager that has its own manner to handle installation of packages.
> But it is not yocto friendly, for instance NPM fetch dependencies in the middle of compilation.
>
> The shrinkwrap file changed its format over npm versions, but npm does not version
> this file, so we can use it properly.
> The actual changes make NPM depencies work with the actual shrinkwrap format.
>
> Signed-off-by: BELOUARGA Mohamed <m.belouarga@technologyandstrategy.com>
> ---
> lib/bb/fetch2/npmsw.py | 25 +++++++++----------------
> 1 file changed, 9 insertions(+), 16 deletions(-)
>
> diff --git a/lib/bb/fetch2/npmsw.py b/lib/bb/fetch2/npmsw.py
> index cc81100b3a..359192a29e 100644
> --- a/lib/bb/fetch2/npmsw.py
> +++ b/lib/bb/fetch2/npmsw.py
> @@ -41,20 +41,15 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> with:
> name = the package name (string)
> params = the package parameters (dictionary)
> - deptree = the package dependency tree (array of strings)
> + destdir = the destination of the package (string)
> """
> - def _walk_deps(deps, deptree):
> - for name in deps:
> - subtree = [*deptree, name]
> - _walk_deps(deps[name].get("dependencies", {}), subtree)
> - if callback is not None:
> - if deps[name].get("dev", False) and not dev:
> - continue
> - elif deps[name].get("bundled", False):
> - continue
> - callback(name, deps[name], subtree)
> -
> - _walk_deps(shrinkwrap.get("dependencies", {}), [])
> + packages = shrinkwrap.get("packages", {})
> +
> + for package in packages:
> + if package != "":
> + name = package.split('node_modules/')[-1]
> + package_infos = packages.get(package, {})
> + callback(name, package_infos, package)
>
> class NpmShrinkWrap(FetchMethod):
> """Class to fetch all package from a shrinkwrap file"""
> @@ -75,12 +70,10 @@ class NpmShrinkWrap(FetchMethod):
> # Resolve the dependencies
> ud.deps = []
>
> - def _resolve_dependency(name, params, deptree):
> + def _resolve_dependency(name, params, destsuffix):
> url = None
> localpath = None
> extrapaths = []
> - destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
> - destsuffix = os.path.join(*destsubdirs)
> unpack = True
>
> integrity = params.get("integrity", None)
> --
> 2.25.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#14827): https://lists.openembedded.org/g/bitbake-devel/message/14827
> Mute This Topic: https://lists.openembedded.org/mt/99314980/3617179
> Group Owner: bitbake-devel+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
Hello, I just made a look on this test: test_npmsw_no_network_no_tarball It seems to me that it is outdated. Latest versions of Nodejs do not use the shrinkwrap that test_npmsw_no_network_no_tarball used.
On 07/06/2023 06:05:26-0700, belouargamohamed@gmail.com wrote: > Hello, > > I just made a look on this test: test_npmsw_no_network_no_tarball > > It seems to me that it is outdated. Latest versions of Nodejs do not use the shrinkwrap that test_npmsw_no_network_no_tarball used. Then, you need to update the test and ideally, add coverage for what you are adding. > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#14833): https://lists.openembedded.org/g/bitbake-devel/message/14833 > Mute This Topic: https://lists.openembedded.org/mt/99314980/3617179 > Group Owner: bitbake-devel+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/lib/bb/fetch2/npmsw.py b/lib/bb/fetch2/npmsw.py index cc81100b3a..359192a29e 100644 --- a/lib/bb/fetch2/npmsw.py +++ b/lib/bb/fetch2/npmsw.py @@ -41,20 +41,15 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False): with: name = the package name (string) params = the package parameters (dictionary) - deptree = the package dependency tree (array of strings) + destdir = the destination of the package (string) """ - def _walk_deps(deps, deptree): - for name in deps: - subtree = [*deptree, name] - _walk_deps(deps[name].get("dependencies", {}), subtree) - if callback is not None: - if deps[name].get("dev", False) and not dev: - continue - elif deps[name].get("bundled", False): - continue - callback(name, deps[name], subtree) - - _walk_deps(shrinkwrap.get("dependencies", {}), []) + packages = shrinkwrap.get("packages", {}) + + for package in packages: + if package != "": + name = package.split('node_modules/')[-1] + package_infos = packages.get(package, {}) + callback(name, package_infos, package) class NpmShrinkWrap(FetchMethod): """Class to fetch all package from a shrinkwrap file""" @@ -75,12 +70,10 @@ class NpmShrinkWrap(FetchMethod): # Resolve the dependencies ud.deps = [] - def _resolve_dependency(name, params, deptree): + def _resolve_dependency(name, params, destsuffix): url = None localpath = None extrapaths = [] - destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] - destsuffix = os.path.join(*destsubdirs) unpack = True integrity = params.get("integrity", None)