From patchwork Fri Jun 27 13:48:47 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65747 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8B4EBC77B7F for ; Fri, 27 Jun 2025 13:49:07 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.14133.1751032137557528471 for ; Fri, 27 Jun 2025 06:48:57 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id EBAEB1A00 for ; Fri, 27 Jun 2025 06:48:39 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CB6DA3F66E for ; Fri, 27 Jun 2025 06:48:56 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Subject: [PATCH v2 6/9] classes/go-mod-update-modules: add class to generate module list Date: Fri, 27 Jun 2025 14:48:47 +0100 Message-ID: <20250627134850.152269-6-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250627134850.152269-1-ross.burton@arm.com> References: <20250627134850.152269-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 27 Jun 2025 13:49:07 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219418 Almost entirely based on the create_go.py module for recipetool by Christian Lindeberg , this instead has the logic inside a class that can be used to update the list of Go modules that are used, both SRC_URI and LICENSE. Integration with devtool upgrade will come shortly, but it needs a bit more work. Signed-off-by: Ross Burton --- .../go-mod-update-modules.bbclass | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 meta/classes-recipe/go-mod-update-modules.bbclass diff --git a/meta/classes-recipe/go-mod-update-modules.bbclass b/meta/classes-recipe/go-mod-update-modules.bbclass new file mode 100644 index 00000000000..6ef7ab553b0 --- /dev/null +++ b/meta/classes-recipe/go-mod-update-modules.bbclass @@ -0,0 +1,152 @@ +addtask do_update_modules after do_configure +do_update_modules[nostamp] = "1" +do_update_modules[network] = "1" + +# This class maintains two files, BPN-go-mods.inc and BPN-licenses.inc. +# +# -go-mods.inc will append SRC_URI with all of the Go modules that are +# dependencies of this recipe. +# +# -licenses.inc will append LICENSE and LIC_FILES_CHKSUM with the found licenses +# in the modules. +# +# These files are machine-generated and should not be modified. + +python do_update_modules() { + import subprocess, tempfile, json, re, urllib.parse + from oe.license import tidy_licenses + from oe.license_finder import find_licenses + + def unescape_path(path): + """Unescape capital letters using exclamation points.""" + return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path) + + def fold_uri(uri): + """Fold URI for sorting shorter module paths before longer.""" + return uri.replace(';', ' ').replace('/', '!') + + def parse_existing_licenses(): + hashes = {} + for url in d.getVar("LIC_FILES_CHKSUM").split(): + (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) + if "spdx" in parm and parm["spdx"] != "Unknown": + hashes[parm["md5"]] = urllib.parse.unquote_plus(parm["spdx"]) + return hashes + + bpn = d.getVar("BPN") + thisdir = d.getVar("THISDIR") + s_dir = d.getVar("S") + + with tempfile.TemporaryDirectory(prefix='go-mod-') as mod_cache_dir: + notice = """ +# This file has been generated by go-mod-update-modules.bbclass +# +# Do not modify it by hand, as the contents will be replaced when +# running the update-modules task. + +""" + + env = dict(os.environ, GOMODCACHE=mod_cache_dir) + + source = d.expand("${WORKDIR}/${GO_SRCURI_DESTSUFFIX}") + output = subprocess.check_output(("go", "mod", "edit", "-json"), cwd=source, env=env, text=True) + go_mod = json.loads(output) + + output = subprocess.check_output(("go", "list", "-json=Dir,Module", "-deps", f"{go_mod['Module']['Path']}/..."), cwd=source, env=env, text=True) + + # + # Licenses + # + + # load hashes from the existing licenses.inc + extra_hashes = parse_existing_licenses() + + # The output of this isn't actually valid JSON, but a series of dicts. + # Wrap in [] and join the dicts with , + # Very frustrating that the json parser in python can't repeatedly + # parse from a stream. + pkgs = json.loads('[' + output.replace('}\n{', '},\n{') + ']') + # Collect licenses for the dependencies. + licenses = set() + lic_files_chksum = [] + lic_files = {} + + for pkg in pkgs: + mod = pkg.get('Module', None) + if not mod or mod.get('Main', False): + continue + + mod_dir = mod['Dir'] + + if mod_dir.startswith(s_dir): + continue + + path = os.path.relpath(mod_dir, mod_cache_dir) + + for license_name, license_file, license_md5 in find_licenses(mod['Dir'], d, first_only=True, extra_hashes=extra_hashes): + lic_files[os.path.join(path, license_file)] = (license_name, license_md5) + + for lic_file in lic_files: + license_name, license_md5 = lic_files[lic_file] + if license_name == "Unknown": + bb.warn(f"Unknown license: {lic_file} {license_md5}") + + licenses.add(lic_files[lic_file][0]) + lic_files_chksum.append( + f'file://pkg/mod/{lic_file};md5={license_md5};spdx={urllib.parse.quote_plus(license_name)}') + + licenses_filename = os.path.join(thisdir, f"{bpn}-licenses.inc") + with open(licenses_filename, "w") as f: + f.write(notice) + f.write(f'LICENSE += "& {" & ".join(tidy_licenses(licenses))}"\n\n') + f.write('LIC_FILES_CHKSUM += "\\\n') + for lic in sorted(lic_files_chksum, key=fold_uri): + f.write(' ' + lic + ' \\\n') + f.write('"\n') + + # + # Sources + # + + # Collect the module cache files downloaded by the go list command as + # the go list command knows best what the go list command needs and it + # needs more files in the module cache than the go install command as + # it doesn't do the dependency pruning mentioned in the Go module + # reference, https://go.dev/ref/mod, for go 1.17 or higher. + src_uris = [] + downloaddir = os.path.join(mod_cache_dir, 'cache', 'download') + for dirpath, _, filenames in os.walk(downloaddir): + # We want to process files under @v directories + path, base = os.path.split(os.path.relpath(dirpath, downloaddir)) + if base != '@v': + continue + + path = unescape_path(path) + zipver = None + for name in filenames: + ver, ext = os.path.splitext(name) + if ext == '.zip': + chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) + src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}') + zipver = ver + break + for name in filenames: + ver, ext = os.path.splitext(name) + if ext == '.mod' and ver != zipver: + chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) + src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}') + + + go_mods_filename = os.path.join(thisdir, f"{bpn}-go-mods.inc") + with open(go_mods_filename, "w") as f: + f.write(notice) + f.write('SRC_URI += "\\\n') + for uri in sorted(src_uris, key=fold_uri): + f.write(' ' + uri + ' \\\n') + f.write('"\n') + + subprocess.check_output(("go", "clean", "-modcache"), cwd=source, env=env, text=True) +} + +# This doesn't work as we need to wipe the inc files first so we don't try looking for LICENSE files that don't yet exist +# RECIPE_UPGRADE_EXTRA_TASKS += "do_update_modules"