[RFC,06/15] npm: Rework to use plain npm commands and avoid npm cache

From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>

Rework the npm class to use plain npm commands and remove the usage of
the npm cache to speed-up builds.

Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
---

 meta/classes/npm.bbclass        | 340 +++++++++++---------------------
 scripts/lib/devtool/standard.py |   6 +-
 2 files changed, 119 insertions(+), 227 deletions(-)

I think a longer commit message would be beneficial here as well, and in
7/8/9/10/11 patches. What has changed? How does the new class work and what
does it do at each step? What problems does it solve compared to the old
implementation? Please keep in mind that getting code to work is only half
the job, the other half is explaining it to other human beings.

Alex

On Wed, 24 Nov 2021 at 15:48, Stefan Herbrechtsmeier <
stefan.herbrechtsmeier-oss@weidmueller.com> wrote:

> From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
>
> Rework the npm class to use plain npm commands and remove the usage of
> the npm cache to speed-up builds.
>
> Signed-off-by: Stefan Herbrechtsmeier <
> stefan.herbrechtsmeier@weidmueller.com>
> ---
>
>  meta/classes/npm.bbclass        | 340 +++++++++++---------------------
>  scripts/lib/devtool/standard.py |   6 +-
>  2 files changed, 119 insertions(+), 227 deletions(-)
>
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index ba50fcac20..cb22dc6998 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -1,7 +1,12 @@
>  # Copyright (C) 2020 Savoir-Faire Linux
>  #
> +# Copyright (C) 2021 Weidmüller Interface GmbH & Co. KG
> +# Author: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
> +#
>  # SPDX-License-Identifier: GPL-2.0-only
>  #
> +# A bbclass to install an npm package and build its dependencies.
> +#
>  # This bbclass builds and installs an npm package to the target. The
> package
>  # sources files should be fetched in the calling recipe by using the
> SRC_URI
>  # variable. The ${S} variable should be updated depending of your fetcher.
> @@ -16,18 +21,29 @@
>  #
>  #  NPM_INSTALL_DEV:
>  #       Set to 1 to also install devDependencies.
> +#
> +#  NPM_BUILD_PRUNE_FOR_PRODUCTION
> +#       Set to 0 to keep installed devDependencies.
>
>  inherit python3native
>
>  DEPENDS:prepend = "nodejs-native "
> -RDEPENDS:${PN}:append:class-target = " nodejs"
> +NPM_RDEPENDS = "nodejs"
> +RDEPENDS:${PN}:append:class-target = " ${NPM_RDEPENDS}"
>
> -EXTRA_OENPM = ""
> +EXTRA_OENPM ?= ""
> +EXTRA_OENPM_BUILD ?= ""
>
>  NPM_INSTALL_DEV ?= "0"
>
>  NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}"
>
> +def npm_flag_dev(value):
> +    if bb.utils.to_boolean(value, False):
> +        return "--also=development"
> +    else:
> +        return "--only=production"
> +
>  def npm_target_arch_map(target_arch):
>      """Maps arch names to npm arch names"""
>      import re
> @@ -43,247 +59,124 @@ def npm_target_arch_map(target_arch):
>
>  NPM_ARCH ?= "${@npm_target_arch_map(d.getVar("TARGET_ARCH"))}"
>
> -NPM_PACKAGE = "${WORKDIR}/npm-package"
>  NPM_CACHE = "${WORKDIR}/npm-cache"
> -NPM_BUILD = "${WORKDIR}/npm-build"
> -
> -def npm_global_configs(d):
> -    """Get the npm global configuration"""
> -    configs = []
> -    # Ensure no network access is done
> -    configs.append(("offline", "true"))
> -    configs.append(("proxy", "http://invalid"))
> -    # Configure the cache directory
> -    configs.append(("cache", d.getVar("NPM_CACHE")))
> -    return configs
> -
> -def npm_pack(env, srcdir, workdir):
> -    """Run 'npm pack' on a specified directory"""
> -    import shlex
> -    cmd = "npm pack %s" % shlex.quote(srcdir)
> -    args = [("ignore-scripts", "true")]
> -    tarball = env.run(cmd, args=args, workdir=workdir).strip("\n")
> -    return os.path.join(workdir, tarball)
> -
> -python npm_do_configure() {
> -    """
> -    Step one: configure the npm cache and the main npm package
> -
> -    Every dependencies have been fetched and patched in the source
> directory.
> -    They have to be packed (this remove unneeded files) and added to the
> npm
> -    cache to be available for the next step.
> -
> -    The main package and its associated manifest file and shrinkwrap file
> have
> -    to be configured to take into account these cached dependencies.
> -    """
> -    import base64
> -    import copy
> -    import json
> -    import re
> -    import shlex
> -    import tempfile
> -    from bb.fetch2.npm import NpmEnvironment
> -    from bb.fetch2.npm import npm_unpack
> -    from bb.fetch2.npmsw import foreach_dependencies
> -    from bb.progress import OutOfProgressHandler
> -
> -    bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
> -    bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> -
> -    env = NpmEnvironment(d, configs=npm_global_configs(d))
> -
> -    def _npm_cache_add(tarball):
> -        """Run 'npm cache add' for a specified tarball"""
> -        cmd = "npm cache add %s" % shlex.quote(tarball)
> -        env.run(cmd)
> -
> -    def _npm_integrity(tarball):
> -        """Return the npm integrity of a specified tarball"""
> -        sha512 = bb.utils.sha512_file(tarball)
> -        return "sha512-" +
> base64.b64encode(bytes.fromhex(sha512)).decode()
> -
> -    def _npm_version(tarball):
> -        """Return the version of a specified tarball"""
> -        regex = r"-(\d+\.\d+\.\d+(-.*)?(\+.*)?)\.tgz"
> -        return re.search(regex, tarball).group(1)
> -
> -    def _npmsw_dependency_dict(orig, deptree):
> -        """
> -        Return the sub dictionary in the 'orig' dictionary corresponding
> to the
> -        'deptree' dependency tree. This function follows the shrinkwrap
> file
> -        format.
> -        """
> -        ptr = orig
> -        for dep in deptree:
> -            if "dependencies" not in ptr:
> -                ptr["dependencies"] = {}
> -            ptr = ptr["dependencies"]
> -            if dep not in ptr:
> -                ptr[dep] = {}
> -            ptr = ptr[dep]
> -        return ptr
> -
> -    # Manage the manifest file and shrinkwrap files
> -    orig_manifest_file = d.expand("${S}/package.json")
> -    orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json")
> -    cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json")
> -    cached_shrinkwrap_file =
> d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json")
> -
> -    with open(orig_manifest_file, "r") as f:
> -        orig_manifest = json.load(f)
> -
> -    cached_manifest = copy.deepcopy(orig_manifest)
> -    cached_manifest.pop("dependencies", None)
> -    cached_manifest.pop("devDependencies", None)
> -
> -    has_shrinkwrap_file = True
> -
> -    try:
> -        with open(orig_shrinkwrap_file, "r") as f:
> -            orig_shrinkwrap = json.load(f)
> -    except IOError:
> -        has_shrinkwrap_file = False
> -
> -    if has_shrinkwrap_file:
> -       cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
> -       cached_shrinkwrap.pop("dependencies", None)
> -
> -    # Manage the dependencies
> -    progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
> -    progress_total = 1 # also count the main package
> -    progress_done = 0
> -
> -    def _count_dependency(name, params, deptree):
> -        nonlocal progress_total
> -        progress_total += 1
> -
> -    def _cache_dependency(name, params, deptree):
> -        destsubdirs = [os.path.join("node_modules", dep) for dep in
> deptree]
> -        destsuffix = os.path.join(*destsubdirs)
> -        with tempfile.TemporaryDirectory() as tmpdir:
> -            # Add the dependency to the npm cache
> -            destdir = os.path.join(d.getVar("S"), destsuffix)
> -            tarball = npm_pack(env, destdir, tmpdir)
> -            _npm_cache_add(tarball)
> -            # Add its signature to the cached shrinkwrap
> -            dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree)
> -            dep["version"] = _npm_version(tarball)
> -            dep["integrity"] = _npm_integrity(tarball)
> -            if params.get("dev", False):
> -                dep["dev"] = True
> -            # Display progress
> -            nonlocal progress_done
> -            progress_done += 1
> -            progress.write("%d/%d" % (progress_done, progress_total))
> -
> -    dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
> -
> -    if has_shrinkwrap_file:
> -        foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> -        foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> -
> -    # Configure the main package
> -    with tempfile.TemporaryDirectory() as tmpdir:
> -        tarball = npm_pack(env, d.getVar("S"), tmpdir)
> -        npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d)
> -
> -    # Configure the cached manifest file and cached shrinkwrap file
> -    def _update_manifest(depkey):
> -        for name in orig_manifest.get(depkey, {}):
> -            version = cached_shrinkwrap["dependencies"][name]["version"]
> -            if depkey not in cached_manifest:
> -                cached_manifest[depkey] = {}
> -            cached_manifest[depkey][name] = version
> -
> -    if has_shrinkwrap_file:
> -        _update_manifest("dependencies")
> -
> -    if dev:
> -        if has_shrinkwrap_file:
> -            _update_manifest("devDependencies")
> -
> -    with open(cached_manifest_file, "w") as f:
> -        json.dump(cached_manifest, f, indent=2)
> -
> -    if has_shrinkwrap_file:
> -        with open(cached_shrinkwrap_file, "w") as f:
> -            json.dump(cached_shrinkwrap, f, indent=2)
> -}
> -
> -python npm_do_compile() {
> -    """
> -    Step two: install the npm package
>
> -    Use the configured main package and the cached dependencies to run the
> -    installation process. The installation is done in a directory which is
> -    not the destination directory yet.
> +NPM_PRUNE_FOR_PRODUCTION ?= "1"
>
> -    A combination of 'npm pack' and 'npm install' is used to ensure that
> the
> -    installed files are actual copies instead of symbolic links (which is
> the
> -    default npm behavior).
> -    """
> -    import shlex
> -    import tempfile
> -    from bb.fetch2.npm import NpmEnvironment
> +B = "${WORKDIR}/build"
>
> -    bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True)
> +NPM_SOURCEPATH ?= "${S}"
>
> -    with tempfile.TemporaryDirectory() as tmpdir:
> -        args = []
> -        configs = npm_global_configs(d)
> +export NPM_CONFIG_GLOBALCONFIG = "${WORKDIR}/npmrc"
> +export NPM_CONFIG_USERCONFIG = "/dev/null"
>
> -        if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False):
> -            configs.append(("also", "development"))
> -        else:
> -            configs.append(("only", "production"))
> -
> -        # Report as many logs as possible for debugging purpose
> -        configs.append(("loglevel", "silly"))
> +oe_runnpm_plain() {
> +    HOME=${B}/root npm "$@"
> +}
>
> -        # Configure the installation to be done globally in the build
> directory
> -        configs.append(("global", "true"))
> -        configs.append(("prefix", d.getVar("NPM_BUILD")))
> +oe_runnpm() {
> +    bbnote npm "$@"
> +    oe_runnpm_plain "$@"
> +}
>
> -        # Add node-gyp configuration
> -        configs.append(("arch", d.getVar("NPM_ARCH")))
> -        configs.append(("release", "true"))
> -        configs.append(("nodedir", d.getVar("NPM_NODEDIR")))
> -        configs.append(("python", d.getVar("PYTHON")))
> +oe_runnpm_pack() {
> +    oe_runnpm_plain pack --ignore-scripts --loglevel=warn "$@"
> +}
>
> -        env = NpmEnvironment(d, configs)
> +oe_runnpm_rebuild() {
> +    arch=$1
> +    shift
>
> -        # Add node-pre-gyp configuration
> -        args.append(("target_arch", d.getVar("NPM_ARCH")))
> -        args.append(("build-from-source", "true"))
> +    # Create symlinks for package executables
> +    # because rebuild doesn't respect the dependency tree
> +    oe_runnpm rebuild --ignore-scripts
>
> -        # Pack and install the main package
> -        tarball = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir)
> -        cmd = "npm install %s %s" % (shlex.quote(tarball),
> d.getVar("EXTRA_OENPM"))
> -        env.run(cmd, args=args)
> +    # Rebuild dependencies
> +    oe_runnpm rebuild \
> +        --arch=${arch} \
> +        --target_arch=${arch} \
> +        --build-from-source \
> +        "$@"
>  }
>
> -npm_do_install() {
> -    # Step three: final install
> -    #
> -    # The previous installation have to be filtered to remove some extra
> files.
> +do_generate_npmrc() {
> +    cat > ${NPM_CONFIG_GLOBALCONFIG} <<EOF
> +cache=${NPM_CACHE}
> +loglevel=silly
> +offline=true
> +package-lock=false
> +proxy=http://invalid
> +; node-gyp
> +arch=${NPM_ARCH}
> +nodedir=${NPM_NODEDIR}
> +python=${PYTHON}
> +release=true
> +EOF
> +}
> +do_generate_npmrc[vardeps] += "NPM_ARCH NPM_CACHE NPM_CONFIG_GLOBALCONFIG
> NPM_NODEDIR PYTHON"
> +addtask generate_npmrc after do_patch before do_populate_cache
>
> -    rm -rf ${D}
> +npm_do_configure() {
> +    :
> +}
>
> -    # Copy the entire lib and bin directories
> -    install -d ${D}/${nonarch_libdir}
> -    cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/.
> ${D}/${nonarch_libdir}
> +npm_do_compile() {
> +    # Create a tarball from main npm package
> +    # to respect ignore files and files property of package.json
> +    cd ${B}/pkg
> +    tarball=$(oe_runnpm_pack ${NPM_SOURCEPATH})
> +    [ -n "${tarball}" ] || exit 1
> +
> +    # Unpack main npm package
> +    mkdir ${B}/lib/node_modules/${BPN}/
> +    tar -xzf ${B}/pkg/$tarball --no-same-owner \
> +        --strip-components=1 -C ${B}/lib/node_modules/${BPN}
> +
> +    # Copy npm dependencies from sources
> +    if [ -d ${S}/node_modules ]; then
> +        # Copy sources without preserve mode and ownership
> +        # to prevent ugly file permissions
> +        mkdir ${B}/lib/node_modules/${BPN}/node_modules
> +        cp -a --no-preserve=mode,ownership \
> +            ${S}/node_modules/. ${B}/lib/node_modules/${BPN}/node_modules/
> +
> +        # Prune dependencies for production
> +        if [ "${NPM_INSTALL_DEV}" = "1" -a "${NPM_PRUNE_FOR_PRODUCTION}"
> = "1" ] ; then
> +            oe_runnpm prune --only=production
> +        fi
> +
> +        # Rebuild dependencies inside node_modules folder
> +        # to create bin links in .bin folder
> +        cd ${B}/lib/node_modules
> +        oe_runnpm_rebuild ${NPM_ARCH} ${EXTRA_OENPM}
> +    fi
> +}
> +do_compile[cleandirs] = "${B}/lib/node_modules ${B}/pkg"
>
> -    if [ -d "${NPM_BUILD}/bin" ]
> -    then
> -        install -d ${D}/${bindir}
> -        cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/.
> ${D}/${bindir}
> +npm_do_install() {
> +    # Copy the entire main node_modules directory
> +    install -d ${D}${nonarch_libdir}
> +    cp -a --no-preserve=ownership ${B}/lib/node_modules \
> +        ${D}${nonarch_libdir}/
> +
> +    # Create symlinks for package executables
> +    if [ -d ${D}${nonarch_libdir}/node_modules/.bin ]; then
> +        install -d ${D}${bindir}
> +        for f in ${D}${nonarch_libdir}/node_modules/.bin/*; do
> +            name="$(basename $f)"
> +            link="$(readlink $f)"
> +            target="$(realpath -m --relative-to ${D}${bindir} \
> +                ${D}${nonarch_libdir}/node_modules/.bin/$link)"
> +            ln -s "$target" "${D}${bindir}/$name"
> +        done
> +        rm -rf ${D}${nonarch_libdir}/node_modules/.bin
>      fi
>
>      # If the package (or its dependencies) uses node-gyp to build native
> addons,
>      # object files, static libraries or other temporary files can be
> hidden in
>      # the lib directory. To reduce the package size and to avoid QA issues
>      # (staticdev with static library files) these files must be removed.
> -    local GYP_REGEX=".*/build/Release/[^/]*.node"
> +    local GYP_REGEX=".*/build.*/Release/[^/]*.node"
>
>      # Remove any node-gyp directory in ${D} to remove temporary build
> files
>      for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}")
> @@ -294,9 +187,9 @@ npm_do_install() {
>      done
>
>      # Copy only the node-gyp release files
> -    for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}")
> +    for GYP_B_FILE in $(find ${B}/pkg -regex "${GYP_REGEX}")
>      do
> -        local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}}
> +        local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${B}/pkg}
>
>          install -d ${GYP_D_FILE%/*}
>          install -m 755 ${GYP_B_FILE} ${GYP_D_FILE}
> @@ -310,6 +203,7 @@ npm_do_install() {
>      # using /usr/lib/node_modules as install directory. Let's make both
> happy.
>      ln -fs node_modules ${D}/${nonarch_libdir}/node
>  }
> +do_install[cleandirs] = "${D}"
>
>  FILES:${PN} += " \
>      ${bindir} \
> diff --git a/scripts/lib/devtool/standard.py
> b/scripts/lib/devtool/standard.py
> index 01fb5ad96f..10dc2b1373 100644
> --- a/scripts/lib/devtool/standard.py
> +++ b/scripts/lib/devtool/standard.py
> @@ -260,10 +260,8 @@ def add(args, config, basepath, workspace):
>                  f.write('}\n')
>
>              if bb.data.inherits_class('npm', rd):
> -                f.write('python do_configure:append() {\n')
> -                f.write('    pkgdir = d.getVar("NPM_PACKAGE")\n')
> -                f.write('    lockfile = os.path.join(pkgdir,
> "singletask.lock")\n')
> -                f.write('    bb.utils.remove(lockfile)\n')
> +                f.write('do_configure:append() {\n')
> +                f.write('    rm -f ${NPM_PACKAGE}/singletask.lock\n')
>                  f.write('}\n')
>
>          # Check if the new layer provides recipes whose priorities have
> been
> --
> 2.20.1
>
>