From patchwork Fri Jun 20 15:14:18 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65355 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A56A5C7115D for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.2671.1750432475738938291 for ; Fri, 20 Jun 2025 08:14:36 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8AEF116F2; Fri, 20 Jun 2025 08:14:15 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 92AB83F673; Fri, 20 Jun 2025 08:14:34 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 01/12] tinfoil: add wait_for decorator and build_file_sync() helper Date: Fri, 20 Jun 2025 16:14:18 +0100 Message-ID: <20250620151429.3210879-1-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219128 The bitbake worker/server IPC is asynchronous, but tinfoil only has functionality to wait for a response on the build_targets() call. Extract the bulk of the "wait for events and handle errors" logic to a standalone wait_for wrapper, which is the build_targets code without the extra_events or event_callback arguments (for now). Then use this to create a build_file_sync() helper that just wraps the existing build_file() with @wait_for. Signed-off-by: Ross Burton --- bitbake/lib/bb/tinfoil.py | 125 +++++++++++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py index f48baeb3342..71c299689cb 100644 --- a/bitbake/lib/bb/tinfoil.py +++ b/bitbake/lib/bb/tinfoil.py @@ -14,7 +14,7 @@ import time import atexit import re from collections import OrderedDict, defaultdict -from functools import partial +from functools import partial, wraps from contextlib import contextmanager import bb.cache @@ -27,6 +27,125 @@ import bb.remotedata from bb.main import setup_bitbake, BitBakeConfigParameters import bb.fetch2 +def wait_for(f): + @wraps(f) + def wrapper(self, *args, **kwds): + # A reasonable set of default events matching up with those we handle below + eventmask = [ + 'bb.event.BuildStarted', + 'bb.event.BuildCompleted', + 'logging.LogRecord', + 'bb.event.NoProvider', + 'bb.command.CommandCompleted', + 'bb.command.CommandFailed', + 'bb.build.TaskStarted', + 'bb.build.TaskFailed', + 'bb.build.TaskSucceeded', + 'bb.build.TaskFailedSilent', + 'bb.build.TaskProgress', + 'bb.runqueue.runQueueTaskStarted', + 'bb.runqueue.sceneQueueTaskStarted', + 'bb.event.ProcessStarted', + 'bb.event.ProcessProgress', + 'bb.event.ProcessFinished', + ] + #if extra_events: + # eventmask.extend(extra_events) + ret = self.set_event_mask(eventmask) + + # Call actual function + ret = f(self, *args, **kwds) + + includelogs = self.config_data.getVar('BBINCLUDELOGS') + loglines = self.config_data.getVar('BBINCLUDELOGS_LINES') + lastevent = time.time() + result = False + # Borrowed from knotty, instead somewhat hackily we use the helper + # as the object to store "shutdown" on + helper = bb.ui.uihelper.BBUIHelper() + helper.shutdown = 0 + parseprogress = None + termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet) + try: + while True: + try: + event = self.wait_event(0.25) + if event: + lastevent = time.time() + #if event_callback and event_callback(event): + # continue + if helper.eventHandler(event): + if isinstance(event, bb.build.TaskFailedSilent): + self.logger.warning("Logfile for failed setscene task is %s" % event.logfile) + elif isinstance(event, bb.build.TaskFailed): + bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter) + continue + if isinstance(event, bb.event.ProcessStarted): + if self.quiet > 1: + continue + parseprogress = bb.ui.knotty.new_progress(event.processname, event.total) + parseprogress.start(False) + continue + if isinstance(event, bb.event.ProcessProgress): + if self.quiet > 1: + continue + if parseprogress: + parseprogress.update(event.progress) + else: + bb.warn("Got ProcessProgress event for something that never started?") + continue + if isinstance(event, bb.event.ProcessFinished): + if self.quiet > 1: + continue + if parseprogress: + parseprogress.finish() + parseprogress = None + continue + if isinstance(event, bb.command.CommandCompleted): + result = True + break + if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)): + self.logger.error(str(event)) + result = False + break + if isinstance(event, logging.LogRecord): + if event.taskpid == 0 or event.levelno > logging.INFO: + self.logger.handle(event) + continue + if isinstance(event, bb.event.NoProvider): + self.logger.error(str(event)) + result = False + break + elif helper.shutdown > 1: + break + termfilter.updateFooter() + if time.time() > (lastevent + (3*60)): + if not self.run_command('ping', handle_events=False): + print("\nUnable to ping server and no events, closing down...\n") + return False + except KeyboardInterrupt: + termfilter.clearFooter() + if helper.shutdown == 1: + print("\nSecond Keyboard Interrupt, stopping...\n") + ret = self.run_command("stateForceShutdown") + if ret and ret[2]: + self.logger.error("Unable to cleanly stop: %s" % ret[2]) + elif helper.shutdown == 0: + print("\nKeyboard Interrupt, closing down...\n") + interrupted = True + ret = self.run_command("stateShutdown") + if ret and ret[2]: + self.logger.error("Unable to cleanly shutdown: %s" % ret[2]) + helper.shutdown = helper.shutdown + 1 + termfilter.clearFooter() + finally: + termfilter.finish() + if helper.failed_tasks: + result = False + return result + + return wrapper + # We need this in order to shut down the connection to the bitbake server, # otherwise the process will never properly exit @@ -700,6 +819,10 @@ class Tinfoil: """ return self.run_command('buildFile', buildfile, task, internal) + @wait_for + def build_file_sync(self, *args): + self.build_file(*args) + def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None): """ Builds the specified targets. This is equivalent to a normal invocation From patchwork Fri Jun 20 15:14:19 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65356 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B29EFC7EE2A for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.2672.1750432476378470028 for ; Fri, 20 Jun 2025 08:14:36 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 5804A176A; Fri, 20 Jun 2025 08:14:16 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 5A0983F673; Fri, 20 Jun 2025 08:14:35 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 02/12] lib/oe/go: document map_arch, and raise an error on unknown architecture Date: Fri, 20 Jun 2025 16:14:19 +0100 Message-ID: <20250620151429.3210879-2-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219129 Add a comment explaining what this function does and where the values come from. If the architecture isn't know, instead of returning an empty string which could fail mysteriously, raise a KeyError so it fails quickly. Signed-off-by: Ross Burton --- meta/lib/oe/go.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meta/lib/oe/go.py b/meta/lib/oe/go.py index dfd957d157a..4559dc63b28 100644 --- a/meta/lib/oe/go.py +++ b/meta/lib/oe/go.py @@ -7,6 +7,10 @@ import re def map_arch(a): + """ + Map our architecture names to Go's GOARCH names. + See https://github.com/golang/go/blob/master/src/internal/syslist/syslist.go for the complete list. + """ if re.match('i.86', a): return '386' elif a == 'x86_64': @@ -31,4 +35,4 @@ def map_arch(a): return 'riscv64' elif a == 'loongarch64': return 'loong64' - return '' + raise KeyError(f"Cannot map architecture {a}") From patchwork Fri Jun 20 15:14:20 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65363 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id EEE8EC7EE31 for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.2673.1750432477091485235 for ; Fri, 20 Jun 2025 08:14:37 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 20D3F22FA; Fri, 20 Jun 2025 08:14:17 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 2AC173F673; Fri, 20 Jun 2025 08:14:36 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 03/12] files/license-hashes.csv: add more hashes Date: Fri, 20 Jun 2025 16:14:20 +0100 Message-ID: <20250620151429.3210879-3-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219130 Add a slew of license hashes harvested by building some Go recipes in meta-oe. Signed-off-by: Ross Burton --- meta/files/license-hashes.csv | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/meta/files/license-hashes.csv b/meta/files/license-hashes.csv index 906660b85df..5e81dbcdec7 100644 --- a/meta/files/license-hashes.csv +++ b/meta/files/license-hashes.csv @@ -3,7 +3,8 @@ 0636e73ff0215e8d672dc4c32c317bb3,GPL-2.0-only 063b5c3ebb5f3aa4c85a2ed18a31fbe7,GPL-2.0-only 0a97f8e4cbaf889d6fa51f84b89a79f6,ISC -0ceb9ff3b27d3a8cf451ca3785d73c71,BSD-3-Clause & MIT +0c241922fc69330e2e5590de114f3bf5,BSD-3-Clause +0ceb9ff3b27d3a8cf451ca3785d73c71,MIT 0dd48ae8103725bd7b401261520cdfbb,BSD-3-Clause 0e46634a01bfef056892949acaea85b1,BSD-3-Clause 1034431802e57486b393d00c5d262b8a,Apache-2.0 @@ -13,16 +14,22 @@ 201414b6610203caed355323b1ab3116,BSD-3-Clause 252890d9eee26aab7b432e8b8a616475,LGPL-2.0-only 2b8c039b2b9a25f0feb4410c4542d346,BSD-2-Clause +2d1c30374313ae40df7772dc92ef9fd5,GPL-3.0-only 2d5025d4aa3495befef8f17206a5b0a1,LGPL-2.1-only 3214f080875748938ba060314b4f727d,LGPL-2.0-only +37acb030ba070680be4a9fcb57f2735a,MIT 385c55653886acac3821999a3ccd17b3,Artistic-1.0 | GPL-2.0-only +38be95f95200434dc208e2ee3dab5081,BSD-3-Clause 393a5ca445f6965873eca0259a17f833,GPL-2.0-only 3b83ef96387f14655fc854ddc3c6bd57,Apache-2.0 3bf50002aefd002f49e7bb854063f7e7,LGPL-2.0-only +3c91c17266710e16afdbb2b6d15c761c,Apache-2.0 3debde09238a8c8e1f6a847e1ec9055b,LGPL-2.1-only 4325afd396febcb659c36b49533135d4,GPL-2.0-only 4c641f2d995c47f5cb08bdb4b5b6ea05,BSD-2-Clause +4e2a8d8f9935d6a766a5879a77ddc24d,Apache-2.0 4ee4feb2b545c2231749e5c54ace343e,BSD-3-Clause +4efd8bdbab9cff00dcd75f2c3ee4e190,BSD-2-Clause 4fbd65380cdd255951079008b364516c,LGPL-2.1-only 50fab24ce589d69af8964fdbfe414c60,BSD-2-Clause 54c7042be62e169199200bc6477f04d1,BSD-3-Clause @@ -30,6 +37,7 @@ 59530bdf33659b29e73d4adb9f9f6552,GPL-2.0-only 5d4950ecb7b26d2c5e4e7b4e0dd74707,BSD-3-Clause 5f30f0716dfdd0d91eb439ebec522ec2,LGPL-2.0-only +6551d79bf661eed41a50157513ee4ad6,BSD-3-Clause 6a6a8e020838b23406c81b19c1d46df6,LGPL-3.0-only 721f23a96ff4161ca3a5f071bbe18108,MIT 7364d1e4653d3584181e9d22d81f275f,CC0-1.0 @@ -41,8 +49,10 @@ 7fbc338309ac38fefcd64b04bb903e34,LGPL-2.1-only 80fa7b56a28e8c902e6af194003220a5,BSD-2-Clause 85d8a977ee9d7c5ab4ac03c9b95431c4,MIT-0 +88073b6dd8ec00fe09da59e0b6dfded1,BSD-3-Clause 88a4355858a1433fea99fae34a44da88,GPL-2.0-only 8bd23871802951c9ad63855151204c2c,BSD-2-Clause +8c6127b79304a5e0a5756d03c7a58766,MIT 8ca43cbc842c2336e835926c2166c28b,GPL-2.0-only 939cce1ec101726fa754e698ac871622,BSD-3-Clause 94d55d512a9ba36caa9b7df079bae19f,GPL-2.0-only @@ -53,8 +63,10 @@ a54a1a6a39e7f9dbb4a23a42f5c7fd1c,Apache-2.0 a651bb3d8b1c412632e28823bb432b40,BSD-3-Clause a6f89e2100d9b6cdffcea4f398e37343,LGPL-2.1-only ad4e9d34a2e966dfe9837f18de03266d,GFDL-1.1-only +ae6de3dee02712a1e55b280671be53bf,BSD-3-Clause b234ee4d69f5fce4486a80fdaf4a4263,GPL-2.0-only b27575459e02221ccef97ec0bfd457ae,Apache-2.0 +b278a92d2c1509760384428817710378,MPL-2.0 b376d29a53c9573006b9970709231431,MIT b5f72aef53d3b2b432702c30b0215666,BSD-3-Clause b66384e7137e41a9b1904ef4d39703b6,Apache-2.0 @@ -63,12 +75,16 @@ bfe1f75d606912a4111c90743d6c7325,MPL-1.1-only c93c0550bd3173f4504b2cbd8991e50b,GPL-2.0-only d014fb11a34eb67dc717fdcfc97e60ed,GFDL-1.2-only d0b68be4a2dc957aaf09144970bc6696,MIT +d0fd9ebda39468b51ff4539c9fbb13a8,BSD-3-Clause d32239bcb673463ab874e80d47fae504,GPL-3.0-only +d44fdeb607e2d2614db9464dbedd4094,MPL-2.0 +d4eb2084b3083d2c275ec52ef4ba9ac1,BSD-3-Clause d7810fab7487fb0aad327b76f1be7cd7,GPL-2.0-only d8045f3b8f929c1cb29a1e3fd737b499,LGPL-2.1-only db979804f025cf55aabec7129cb671ed,LGPL-2.0-only e020ca655b06c112def28e597ab844f1,GFDL-1.3-only e659f77bfd9002659e112d0d3d59b2c1,BSD-2-Clause +eb1e647870add0502f8f010b19de32af,AGPL-3.0-or-later eb723b61539feef013de476e68b5c50a,GPL-2.0-only ebb5c50ab7cab4baeffba14977030c07,GPL-2.0-only efe2cb9a35826992b9df68224e3c2628,EPL-1.0 From patchwork Fri Jun 20 15:14:21 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65365 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 07AFAC7EE30 for ; Fri, 20 Jun 2025 15:14:46 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2668.1750432478038425752 for ; Fri, 20 Jun 2025 08:14:38 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E3A6D16F2; Fri, 20 Jun 2025 08:14:17 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id E5DF33F673; Fri, 20 Jun 2025 08:14:36 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 04/12] oeqa/core/case: add file exists assertion Date: Fri, 20 Jun 2025 16:14:21 +0100 Message-ID: <20250620151429.3210879-4-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:46 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219131 Add assertFileExists() to simply tests that want to check that a file exists. Signed-off-by: Ross Burton --- meta/lib/oeqa/core/case.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py index bc4446a938e..ad5524a7142 100644 --- a/meta/lib/oeqa/core/case.py +++ b/meta/lib/oeqa/core/case.py @@ -5,6 +5,7 @@ # import base64 +import os import zlib import unittest @@ -57,6 +58,13 @@ class OETestCase(unittest.TestCase): d.tearDownDecorator() self.tearDownMethod() + def assertFileExists(self, filename, msg=None): + """ + Test that filename exists. If it does not, the test will fail. + """ + if not os.path.exists(filename): + self.fail(msg or "%s does not exist" % filename) + class OEPTestResultTestCase: """ Mix-in class to provide functions to make interacting with extraresults for From patchwork Fri Jun 20 15:14:22 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65362 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id E296BC7EE2F for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2669.1750432478790346633 for ; Fri, 20 Jun 2025 08:14:38 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id B3CF0176A; Fri, 20 Jun 2025 08:14:18 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id B5F2C3F673; Fri, 20 Jun 2025 08:14:37 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 05/12] recipetool: create: Support creating extra files named after the recipe Date: Fri, 20 Jun 2025 16:14:22 +0100 Message-ID: <20250620151429.3210879-5-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219132 From: Peter Kjellerstedt Signed-off-by: Peter Kjellerstedt --- scripts/lib/recipetool/create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 3c6ef6719fa..b2101f39325 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -825,7 +825,8 @@ def create_recipe(args): extraoutdir = os.path.join(os.path.dirname(outfile), pn) bb.utils.mkdirhier(extraoutdir) for destfn, extrafile in extrafiles.items(): - shutil.move(extrafile, os.path.join(extraoutdir, destfn)) + fn = destfn.format(pn=pn, pv=realpv) + shutil.move(extrafile, os.path.join(extraoutdir, fn)) lines = lines_before lines_before = [] From patchwork Fri Jun 20 15:14:23 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65361 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id C85F0C7EE2E for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2670.1750432479360482781 for ; Fri, 20 Jun 2025 08:14:39 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 7C60516F2; Fri, 20 Jun 2025 08:14:19 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 861A93F673; Fri, 20 Jun 2025 08:14:38 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 06/12] recipetool: allow recipe create handlers to specify bitbake tasks to run Date: Fri, 20 Jun 2025 16:14:23 +0100 Message-ID: <20250620151429.3210879-6-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219133 When creating a recipe there can be cases where there is a class that does some of the recipe creation (such as cargo-update-recipe-crates). To avoid duplication of code, look for run_task assignments in the extravalues dictionary returned by the handler, and if it is set then call that task after writing the recipe. Signed-off-by: Ross Burton --- scripts/lib/recipetool/create.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index b2101f39325..f0584d84701 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -765,6 +765,7 @@ def create_recipe(args): extrafiles = extravalues.pop('extrafiles', {}) extra_pn = extravalues.pop('PN', None) extra_pv = extravalues.pop('PV', None) + run_task = extravalues.pop('run_task', None) if extra_pv and not realpv: realpv = extra_pv @@ -919,6 +920,10 @@ def create_recipe(args): log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool) tinfoil.modified_files() + if run_task: + logger.info("Running task %s" % run_task) + tinfoil.build_file_sync(outfile, run_task) + if tempsrc: if args.keep_temp: logger.info('Preserving temporary directory %s' % tempsrc) From patchwork Fri Jun 20 15:14:24 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65364 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id D5F04C7EE2C for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2672.1750432480216079078 for ; Fri, 20 Jun 2025 08:14:40 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 62B00176A; Fri, 20 Jun 2025 08:14:20 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 4ECA03F673; Fri, 20 Jun 2025 08:14:39 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 07/12] recipetool: create_go: Use gomod fetcher instead of go mod vendor Date: Fri, 20 Jun 2025 16:14:24 +0100 Message-ID: <20250620151429.3210879-7-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219134 From: Christian Lindeberg Use the go-mod bbclass together with the gomod fetcher instead of the go-vendor bbclass. Signed-off-by: Christian Lindeberg --- scripts/lib/recipetool/create_go.py | 731 ++++------------------------ 1 file changed, 104 insertions(+), 627 deletions(-) diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py index 5cc53931f00..3e9fc857842 100644 --- a/scripts/lib/recipetool/create_go.py +++ b/scripts/lib/recipetool/create_go.py @@ -10,48 +10,31 @@ # -from collections import namedtuple -from enum import Enum -from html.parser import HTMLParser from recipetool.create import RecipeHandler, handle_license_vars -from recipetool.create import find_licenses, tidy_licenses, fixup_license -from recipetool.create import determine_from_url -from urllib.error import URLError, HTTPError +from recipetool.create import find_licenses import bb.utils import json import logging import os import re -import subprocess import sys -import shutil import tempfile import urllib.parse import urllib.request -GoImport = namedtuple('GoImport', 'root vcs url suffix') logger = logging.getLogger('recipetool') -CodeRepo = namedtuple( - 'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor') tinfoil = None -# Regular expression to parse pseudo semantic version -# see https://go.dev/ref/mod#pseudo-versions -re_pseudo_semver = re.compile( - r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P\d{14})-(?P[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$") -# Regular expression to parse semantic version -re_semver = re.compile( - r"^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$") - def tinfoil_init(instance): global tinfoil tinfoil = instance + class GoRecipeHandler(RecipeHandler): """Class to handle the go recipe creation""" @@ -83,577 +66,96 @@ class GoRecipeHandler(RecipeHandler): return bindir - def __resolve_repository_static(self, modulepath): - """Resolve the repository in a static manner - - The method is based on the go implementation of - `repoRootFromVCSPaths` in - https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go - """ - - url = urllib.parse.urlparse("https://" + modulepath) - req = urllib.request.Request(url.geturl()) - - try: - resp = urllib.request.urlopen(req) - # Some modulepath are just redirects to github (or some other vcs - # hoster). Therefore, we check if this modulepath redirects to - # somewhere else - if resp.geturl() != url.geturl(): - bb.debug(1, "%s is redirectred to %s" % - (url.geturl(), resp.geturl())) - url = urllib.parse.urlparse(resp.geturl()) - modulepath = url.netloc + url.path - - except URLError as url_err: - # This is probably because the module path - # contains the subdir and major path. Thus, - # we ignore this error for now - logger.debug( - 1, "Failed to fetch page from [%s]: %s" % (url, str(url_err))) - - host, _, _ = modulepath.partition('/') - - class vcs(Enum): - pathprefix = "pathprefix" - regexp = "regexp" - type = "type" - repo = "repo" - check = "check" - schemelessRepo = "schemelessRepo" - - # GitHub - vcsGitHub = {} - vcsGitHub[vcs.pathprefix] = "github.com" - vcsGitHub[vcs.regexp] = re.compile( - r'^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$') - vcsGitHub[vcs.type] = "git" - vcsGitHub[vcs.repo] = "https://\\g" - - # Bitbucket - vcsBitbucket = {} - vcsBitbucket[vcs.pathprefix] = "bitbucket.org" - vcsBitbucket[vcs.regexp] = re.compile( - r'^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P[A-Za-z0-9_.\-]+))*$') - vcsBitbucket[vcs.type] = "git" - vcsBitbucket[vcs.repo] = "https://\\g" - - # IBM DevOps Services (JazzHub) - vcsIBMDevOps = {} - vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git" - vcsIBMDevOps[vcs.regexp] = re.compile( - r'^(?Phub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$') - vcsIBMDevOps[vcs.type] = "git" - vcsIBMDevOps[vcs.repo] = "https://\\g" - - # Git at Apache - vcsApacheGit = {} - vcsApacheGit[vcs.pathprefix] = "git.apache.org" - vcsApacheGit[vcs.regexp] = re.compile( - r'^(?Pgit\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P[A-Za-z0-9_.\-]+))*$') - vcsApacheGit[vcs.type] = "git" - vcsApacheGit[vcs.repo] = "https://\\g" - - # Git at OpenStack - vcsOpenStackGit = {} - vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org" - vcsOpenStackGit[vcs.regexp] = re.compile( - r'^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P[A-Za-z0-9_.\-]+))*$') - vcsOpenStackGit[vcs.type] = "git" - vcsOpenStackGit[vcs.repo] = "https://\\g" - - # chiselapp.com for fossil - vcsChiselapp = {} - vcsChiselapp[vcs.pathprefix] = "chiselapp.com" - vcsChiselapp[vcs.regexp] = re.compile( - r'^(?Pchiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$') - vcsChiselapp[vcs.type] = "fossil" - vcsChiselapp[vcs.repo] = "https://\\g" - - # General syntax for any server. - # Must be last. - vcsGeneralServer = {} - vcsGeneralServer[vcs.regexp] = re.compile( - "(?P(?P([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?Pbzr|fossil|git|hg|svn))(/~?(?P[A-Za-z0-9_.\\-]+))*$") - vcsGeneralServer[vcs.schemelessRepo] = True - - vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, - vcsApacheGit, vcsOpenStackGit, vcsChiselapp, - vcsGeneralServer] - - if modulepath.startswith("example.net") or modulepath == "rsc.io": - logger.warning("Suspicious module path %s" % modulepath) - return None - if modulepath.startswith("http:") or modulepath.startswith("https:"): - logger.warning("Import path should not start with %s %s" % - ("http", "https")) - return None - - rootpath = None - vcstype = None - repourl = None - suffix = None - - for srv in vcsPaths: - m = srv[vcs.regexp].match(modulepath) - if vcs.pathprefix in srv: - if host == srv[vcs.pathprefix]: - rootpath = m.group('root') - vcstype = srv[vcs.type] - repourl = m.expand(srv[vcs.repo]) - suffix = m.group('suffix') - break - elif m and srv[vcs.schemelessRepo]: - rootpath = m.group('root') - vcstype = m[vcs.type] - repourl = m[vcs.repo] - suffix = m.group('suffix') - break - - return GoImport(rootpath, vcstype, repourl, suffix) - - def __resolve_repository_dynamic(self, modulepath): - """Resolve the repository root in a dynamic manner. - - The method is based on the go implementation of - `repoRootForImportDynamic` in - https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go - """ - url = urllib.parse.urlparse("https://" + modulepath) - - class GoImportHTMLParser(HTMLParser): - - def __init__(self): - super().__init__() - self.__srv = {} - - def handle_starttag(self, tag, attrs): - if tag == 'meta' and list( - filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)): - content = list( - filter(lambda a: (a[0] == 'content'), attrs)) - if content: - srv = content[0][1].split() - self.__srv[srv[0]] = srv - - def go_import(self, modulepath): - if modulepath in self.__srv: - srv = self.__srv[modulepath] - return GoImport(srv[0], srv[1], srv[2], None) - return None - - url = url.geturl() + "?go-get=1" - req = urllib.request.Request(url) - - try: - body = urllib.request.urlopen(req).read() - except HTTPError as http_err: - logger.warning( - "Unclean status when fetching page from [%s]: %s", url, str(http_err)) - body = http_err.fp.read() - except URLError as url_err: - logger.warning( - "Failed to fetch page from [%s]: %s", url, str(url_err)) - return None - - parser = GoImportHTMLParser() - parser.feed(body.decode('utf-8')) - parser.close() - - return parser.go_import(modulepath) - - def __resolve_from_golang_proxy(self, modulepath, version): - """ - Resolves repository data from golang proxy - """ - url = urllib.parse.urlparse("https://proxy.golang.org/" - + modulepath - + "/@v/" - + version - + ".info") - - # Transform url to lower case, golang proxy doesn't like mixed case - req = urllib.request.Request(url.geturl().lower()) - - try: - resp = urllib.request.urlopen(req) - except URLError as url_err: - logger.warning( - "Failed to fetch page from [%s]: %s", url, str(url_err)) - return None - - golang_proxy_res = resp.read().decode('utf-8') - modinfo = json.loads(golang_proxy_res) - - if modinfo and 'Origin' in modinfo: - origin = modinfo['Origin'] - _root_url = urllib.parse.urlparse(origin['URL']) - - # We normalize the repo URL since we don't want the scheme in it - _subdir = origin['Subdir'] if 'Subdir' in origin else None - _root, _, _ = self.__split_path_version(modulepath) - if _subdir: - _root = _root[:-len(_subdir)].strip('/') - - _commit = origin['Hash'] - _vcs = origin['VCS'] - return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit) - - return None - - def __resolve_repository(self, modulepath): - """ - Resolves src uri from go module-path - """ - repodata = self.__resolve_repository_static(modulepath) - if not repodata or not repodata.url: - repodata = self.__resolve_repository_dynamic(modulepath) - if not repodata or not repodata.url: - logger.error( - "Could not resolve repository for module path '%s'" % modulepath) - # There is no way to recover from this - sys.exit(14) - if repodata: - logger.debug(1, "Resolved download path for import '%s' => %s" % ( - modulepath, repodata.url)) - return repodata - - def __split_path_version(self, path): - i = len(path) - dot = False - for j in range(i, 0, -1): - if path[j - 1] < '0' or path[j - 1] > '9': - break - if path[j - 1] == '.': - dot = True - break - i = j - 1 - - if i <= 1 or i == len( - path) or path[i - 1] != 'v' or path[i - 2] != '/': - return path, "", True - - prefix, pathMajor = path[:i - 2], path[i - 2:] - if dot or len( - pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1": - return path, "", False - - return prefix, pathMajor, True - - def __get_path_major(self, pathMajor): - if not pathMajor: - return "" - - if pathMajor[0] != '/' and pathMajor[0] != '.': - logger.error( - "pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor) - - if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"): - pathMajor = pathMajor[:len("-unstable") - 2] - - return pathMajor[1:] - - def __build_coderepo(self, repo, path): - codedir = "" - pathprefix, pathMajor, _ = self.__split_path_version(path) - if repo.root == path: - pathprefix = path - elif path.startswith(repo.root): - codedir = pathprefix[len(repo.root):].strip('/') - - pseudoMajor = self.__get_path_major(pathMajor) - - logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'", - repo.root, codedir, pathprefix, pathMajor, pseudoMajor) - - return CodeRepo(path, repo.root, codedir, - pathMajor, pathprefix, pseudoMajor) - - def __resolve_version(self, repo, path, version): - hash = None - coderoot = self.__build_coderepo(repo, path) - - def vcs_fetch_all(): - tmpdir = tempfile.mkdtemp() - clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir) - bb.process.run(clone_cmd) - log_cmd = "git log --all --pretty='%H %d' --decorate=short" - output, _ = bb.process.run( - log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir) - bb.utils.prunedir(tmpdir) - return output.strip().split('\n') - - def vcs_fetch_remote(tag): - # add * to grab ^{} - refs = {} - ls_remote_cmd = "git ls-remote -q --tags {} {}*".format( - repo.url, tag) - output, _ = bb.process.run(ls_remote_cmd) - output = output.strip().split('\n') - for line in output: - f = line.split(maxsplit=1) - if len(f) != 2: - continue - - for prefix in ["HEAD", "refs/heads/", "refs/tags/"]: - if f[1].startswith(prefix): - refs[f[1][len(prefix):]] = f[0] - - for key, hash in refs.items(): - if key.endswith(r"^{}"): - refs[key.strip(r"^{}")] = hash - - return refs[tag] - - m_pseudo_semver = re_pseudo_semver.match(version) - - if m_pseudo_semver: - remote_refs = vcs_fetch_all() - short_commit = m_pseudo_semver.group('commithash') - for l in remote_refs: - r = l.split(maxsplit=1) - sha1 = r[0] if len(r) else None - if not sha1: - logger.error( - "Ups: could not resolve abbref commit for %s" % short_commit) - - elif sha1.startswith(short_commit): - hash = sha1 - break - else: - m_semver = re_semver.match(version) - if m_semver: - - def get_sha1_remote(re): - rsha1 = None - for line in remote_refs: - # Split lines of the following format: - # 22e90d9b964610628c10f673ca5f85b8c2a2ca9a (tag: sometag) - lineparts = line.split(maxsplit=1) - sha1 = lineparts[0] if len(lineparts) else None - refstring = lineparts[1] if len( - lineparts) == 2 else None - if refstring: - # Normalize tag string and split in case of multiple - # regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...) - refs = refstring.strip('(), ').split(',') - for ref in refs: - if re.match(ref.strip()): - rsha1 = sha1 - return rsha1 - - semver = "v" + m_semver.group('major') + "."\ - + m_semver.group('minor') + "."\ - + m_semver.group('patch') \ - + (("-" + m_semver.group('prerelease')) - if m_semver.group('prerelease') else "") - - tag = os.path.join( - coderoot.codeDir, semver) if coderoot.codeDir else semver - - # probe tag using 'ls-remote', which is faster than fetching - # complete history - hash = vcs_fetch_remote(tag) - if not hash: - # backup: fetch complete history - remote_refs = vcs_fetch_all() - hash = get_sha1_remote( - re.compile(fr"(tag:|HEAD ->) ({tag})")) - - logger.debug( - "Resolving commit for tag '%s' -> '%s'", tag, hash) - return hash - - def __generate_srcuri_inline_fcn(self, path, version, replaces=None): - """Generate SRC_URI functions for go imports""" - - logger.info("Resolving repository for module %s", path) - # First try to resolve repo and commit from golang proxy - # Most info is already there and we don't have to go through the - # repository or even perform the version resolve magic - golang_proxy_info = self.__resolve_from_golang_proxy(path, version) - if golang_proxy_info: - repo = golang_proxy_info[0] - commit = golang_proxy_info[1] - else: - # Fallback - # Resolve repository by 'hand' - repo = self.__resolve_repository(path) - commit = self.__resolve_version(repo, path, version) - - url = urllib.parse.urlparse(repo.url) - repo_url = url.netloc + url.path - - coderoot = self.__build_coderepo(repo, path) - - inline_fcn = "${@go_src_uri(" - inline_fcn += f"'{repo_url}','{version}'" - if repo_url != path: - inline_fcn += f",path='{path}'" - if coderoot.codeDir: - inline_fcn += f",subdir='{coderoot.codeDir}'" - if repo.vcs != 'git': - inline_fcn += f",vcs='{repo.vcs}'" - if replaces: - inline_fcn += f",replaces='{replaces}'" - if coderoot.pathMajor: - inline_fcn += f",pathmajor='{coderoot.pathMajor}'" - inline_fcn += ")}" - - return inline_fcn, commit - - def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d): - - import re - src_uris = [] - src_revs = [] - - def generate_src_rev(path, version, commithash): - src_rev = f"# {path}@{version} => {commithash}\n" - # Ups...maybe someone manipulated the source repository and the - # version or commit could not be resolved. This is a sign of - # a) the supply chain was manipulated (bad) - # b) the implementation for the version resolving didn't work - # anymore (less bad) - if not commithash: - src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" - src_rev += f"#!!! Could not resolve version !!!\n" - src_rev += f"#!!! Possible supply chain attack !!!\n" - src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" - src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\"" - - return src_rev - - # we first go over replacement list, because we are essentialy - # interested only in the replaced path - if go_mod['Replace']: - for replacement in go_mod['Replace']: - oldpath = replacement['Old']['Path'] - path = replacement['New']['Path'] - version = '' - if 'Version' in replacement['New']: - version = replacement['New']['Version'] - - if os.path.exists(os.path.join(srctree, path)): - # the module refers to the local path, remove it from requirement list - # because it's a local module - go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath] - else: - # Replace the path and the version, so we don't iterate replacement list anymore - for require in go_mod['Require']: - if require['Path'] == oldpath: - require.update({'Path': path, 'Version': version}) - break - - for require in go_mod['Require']: - path = require['Path'] - version = require['Version'] - - inline_fcn, commithash = self.__generate_srcuri_inline_fcn( - path, version) - src_uris.append(inline_fcn) - src_revs.append(generate_src_rev(path, version, commithash)) - - # strip version part from module URL /vXX - baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) - pn, _ = determine_from_url(baseurl) - go_mods_basename = "%s-modules.inc" % pn - - go_mods_filename = os.path.join(localfilesdir, go_mods_basename) - with open(go_mods_filename, "w") as f: - # We introduce this indirection to make the tests a little easier - f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n") - f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n") - for uri in src_uris: - f.write(" " + uri + " \\\n") - f.write("\"\n\n") - for rev in src_revs: - f.write(rev + "\n") - - extravalues['extrafiles'][go_mods_basename] = go_mods_filename - - def __go_run_cmd(self, cmd, cwd, d): - return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')), - shell=True, cwd=cwd) - - def __go_native_version(self, d): - stdout, _ = self.__go_run_cmd("go version", None, d) - m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout) - major = int(m.group(2)) - minor = int(m.group(3)) - patch = int(m.group(4)) - - return major, minor, patch - - def __go_mod_patch(self, srctree, localfilesdir, extravalues, d): - - patchfilename = "go.mod.patch" - go_native_version_major, go_native_version_minor, _ = self.__go_native_version( - d) - self.__go_run_cmd("go mod tidy -go=%d.%d" % - (go_native_version_major, go_native_version_minor), srctree, d) - stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) - - # Create patch in order to upgrade go version - self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d) - # Restore original state - self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d) - - go_mod = json.loads(stdout) - tmpfile = os.path.join(localfilesdir, patchfilename) - shutil.move(os.path.join(srctree, patchfilename), tmpfile) - - extravalues['extrafiles'][patchfilename] = tmpfile - - return go_mod, patchfilename - - def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d): - # Perform vendoring to retrieve the correct modules.txt - tmp_vendor_dir = tempfile.mkdtemp() - - # -v causes to go to print modules.txt to stderr - _, stderr = self.__go_run_cmd( - "go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d) - - modules_txt_basename = "modules.txt" - modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename) - with open(modules_txt_filename, "w") as f: - f.write(stderr) - - extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename - - licenses = [] + @staticmethod + def __unescape_path(path): + """Unescape capital letters using exclamation points.""" + return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path) + + @staticmethod + def __fold_uri(uri): + """Fold URI for sorting shorter module paths before longer.""" + return uri.replace(';', ' ').replace('/', '!') + + @staticmethod + def __go_run_cmd(cmd, cwd, d): + env = dict(os.environ, PATH=d.getVar('PATH'), GOMODCACHE=d.getVar('GOMODCACHE')) + return bb.process.run(cmd, env=env, shell=True, cwd=cwd) + + def __go_mod(self, go_mod, srctree, localfilesdir, extravalues, d): + moddir = d.getVar('GOMODCACHE') + + # List main packages and their dependencies with the go list command. + stdout, _ = self.__go_run_cmd(f"go list -json=Dir,Module -deps {go_mod['Module']['Path']}/...", srctree, d) + pkgs = json.loads('[' + stdout.replace('}\n{', '},\n{') + ']') + + # Collect licenses for the dependencies. + licenses = set() lic_files_chksum = [] - licvalues = find_licenses(tmp_vendor_dir, d) - shutil.rmtree(tmp_vendor_dir) + lic_files = {} + for pkg in pkgs: + # TODO: If the package is in a subdirectory with its own license + # files then report those istead of the license files found in the + # module root directory. + mod = pkg.get('Module', None) + if not mod or mod.get('Main', False): + continue + path = os.path.relpath(mod['Dir'], moddir) + for lic in find_licenses(mod['Dir'], d): + lic_files[os.path.join(path, lic[1])] = (lic[0], lic[2]) - if licvalues: - for licvalue in licvalues: - license = licvalue[0] - lics = tidy_licenses(fixup_license(license)) - lics = [lic for lic in lics if lic not in licenses] - if len(lics): - licenses.extend(lics) - lic_files_chksum.append( - 'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2])) + for lic_file in lic_files: + licenses.add(lic_files[lic_file][0]) + lic_files_chksum.append( + f'file://pkg/mod/{lic_file};md5={lic_files[lic_file][1]}') - # strip version part from module URL /vXX - baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) - pn, _ = determine_from_url(baseurl) - licenses_basename = "%s-licenses.inc" % pn + # Collect the module cache files downloaded by the go list command as + # the go list command knows best what the go list command needs and it + # needs more files in the module cache than the go install command as + # it doesn't do the dependency pruning mentioned in the Go module + # reference, https://go.dev/ref/mod, for go 1.17 or higher. + src_uris = [] + downloaddir = os.path.join(moddir, 'cache', 'download') + for dirpath, _, filenames in os.walk(downloaddir): + path, base = os.path.split(os.path.relpath(dirpath, downloaddir)) + if base != '@v': + continue + path = self.__unescape_path(path) + zipver = None + for name in filenames: + ver, ext = os.path.splitext(name) + if ext == '.zip': + chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) + src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}') + zipver = ver + break + for name in filenames: + ver, ext = os.path.splitext(name) + if ext == '.mod' and ver != zipver: + chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) + src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}') + self.__go_run_cmd("go clean -modcache", srctree, d) + + licenses_basename = "{pn}-licenses.inc" licenses_filename = os.path.join(localfilesdir, licenses_basename) with open(licenses_filename, "w") as f: - f.write("GO_MOD_LICENSES = \"%s\"\n\n" % - ' & '.join(sorted(licenses, key=str.casefold))) - # We introduce this indirection to make the tests a little easier - f.write("LIC_FILES_CHKSUM += \"${VENDORED_LIC_FILES_CHKSUM}\"\n") - f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n") - for lic in lic_files_chksum: - f.write(" " + lic + " \\\n") - f.write("\"\n") + f.write(f'GO_MOD_LICENSES = "{" & ".join(sorted(licenses))}"\n\n') + f.write('LIC_FILES_CHKSUM += "\\\n') + for lic in sorted(lic_files_chksum, key=self.__fold_uri): + f.write(' ' + lic + ' \\\n') + f.write('"\n') - extravalues['extrafiles'][licenses_basename] = licenses_filename + extravalues['extrafiles'][f"../{licenses_basename}"] = licenses_filename + + go_mods_basename = "{pn}-go-mods.inc" + go_mods_filename = os.path.join(localfilesdir, go_mods_basename) + with open(go_mods_filename, "w") as f: + f.write('SRC_URI += "\\\n') + for uri in sorted(src_uris, key=self.__fold_uri): + f.write(' ' + uri + ' \\\n') + f.write('"\n') + + extravalues['extrafiles'][f"../{go_mods_basename}"] = go_mods_filename def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): @@ -672,56 +174,30 @@ class GoRecipeHandler(RecipeHandler): d.prependVar('PATH', '%s:' % go_bindir) handled.append('buildsystem') - classes.append("go-vendor") + classes.append("go-mod") + + tmp_mod_dir = tempfile.mkdtemp(prefix='go-mod-') + d.setVar('GOMODCACHE', tmp_mod_dir) stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) - go_mod = json.loads(stdout) - go_import = go_mod['Module']['Path'] - go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go']) - go_version_major = int(go_version_match.group(1)) - go_version_minor = int(go_version_match.group(2)) - src_uris = [] + go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path']) localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') extravalues.setdefault('extrafiles', {}) - # Use an explicit name determined from the module name because it - # might differ from the actual URL for replaced modules - # strip version part from module URL /vXX - baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path']) - pn, _ = determine_from_url(baseurl) - - # go.mod files with version < 1.17 may not include all indirect - # dependencies. Thus, we have to upgrade the go version. - if go_version_major == 1 and go_version_minor < 17: - logger.warning( - "go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.") - go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir, - extravalues, d) - src_uris.append( - "file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename)) - - # Check whether the module is vendored. If so, we have nothing to do. - # Otherwise we gather all dependencies and add them to the recipe - if not os.path.exists(os.path.join(srctree, "vendor")): - - # Write additional $BPN-modules.inc file - self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d) - lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"") - lines_before.append("require %s-licenses.inc" % (pn)) - - self.__rewrite_src_uri(lines_before, ["file://modules.txt"]) - - self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d) - lines_before.append("require %s-modules.inc" % (pn)) + # Write the ${BPN}-licenses.inc and ${BPN}-go-mods.inc files + self.__go_mod(go_mod, srctree, localfilesdir, extravalues, d) # Do generic license handling handle_license_vars(srctree, lines_before, handled, extravalues, d) - self.__rewrite_lic_uri(lines_before) + self.__rewrite_lic_vars(lines_before) - lines_before.append("GO_IMPORT = \"{}\"".format(baseurl)) - lines_before.append("SRCREV_FORMAT = \"${BPN}\"") + self.__rewrite_src_uri(lines_before) + + lines_before.append('require ${BPN}-licenses.inc') + lines_before.append('require ${BPN}-go-mods.inc') + lines_before.append(f'GO_IMPORT = "{go_import}"') def __update_lines_before(self, updated, newlines, lines_before): if updated: @@ -733,9 +209,11 @@ class GoRecipeHandler(RecipeHandler): lines_before.append(line) return updated - def __rewrite_lic_uri(self, lines_before): + def __rewrite_lic_vars(self, lines_before): def varfunc(varname, origvalue, op, newlines): + if varname == 'LICENSE': + return ' & '.join((origvalue, '${GO_MOD_LICENSES}')), None, -1, True if varname == 'LIC_FILES_CHKSUM': new_licenses = [] licenses = origvalue.split('\\') @@ -757,15 +235,14 @@ class GoRecipeHandler(RecipeHandler): return origvalue, None, 0, True updated, newlines = bb.utils.edit_metadata( - lines_before, ['LIC_FILES_CHKSUM'], varfunc) + lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc) return self.__update_lines_before(updated, newlines, lines_before) - def __rewrite_src_uri(self, lines_before, additional_uris = []): + def __rewrite_src_uri(self, lines_before): def varfunc(varname, origvalue, op, newlines): if varname == 'SRC_URI': - src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"] - src_uri.extend(additional_uris) + src_uri = ['git://${GO_IMPORT};protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}'] return src_uri, None, -1, True return origvalue, None, 0, True From patchwork Fri Jun 20 15:14:25 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65357 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id BF4DFC7EE2D for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2674.1750432481034225125 for ; Fri, 20 Jun 2025 08:14:41 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 2B74116F2; Fri, 20 Jun 2025 08:14:21 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 3516D3F673; Fri, 20 Jun 2025 08:14:40 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 08/12] oe/licenses: move tidy_licenses from recipetool Date: Fri, 20 Jun 2025 16:14:25 +0100 Message-ID: <20250620151429.3210879-8-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219135 This function, to tidy a license string, is useful outside of recipetool so move it to oe.license. Signed-off-by: Ross Burton --- meta/lib/oe/license.py | 15 +++++++++++++++ scripts/lib/recipetool/create.py | 11 +---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py index 6f882c3812c..6e55fa1e7f6 100644 --- a/meta/lib/oe/license.py +++ b/meta/lib/oe/license.py @@ -462,3 +462,18 @@ def skip_incompatible_package_licenses(d, pkgs): skipped_pkgs[pkg] = incompatible_lic return skipped_pkgs + +def tidy_licenses(value): + """ + Flat, split and sort licenses. + """ + from oe.license import flattened_licenses + + def _choose(a, b): + str_a, str_b = sorted((" & ".join(a), " & ".join(b)), key=str.casefold) + return ["(%s | %s)" % (str_a, str_b)] + + if not isinstance(value, str): + value = " & ".join(value) + + return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold) diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index f0584d84701..0a6f59810ce 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -18,6 +18,7 @@ from urllib.parse import urlparse, urldefrag, urlsplit import hashlib import bb.fetch2 logger = logging.getLogger('recipetool') +from oe.license import tidy_licenses from oe.license_finder import find_licenses tinfoil = None @@ -951,16 +952,6 @@ def fixup_license(value): return '(' + value + ')' return value -def tidy_licenses(value): - """Flat, split and sort licenses""" - from oe.license import flattened_licenses - def _choose(a, b): - str_a, str_b = sorted((" & ".join(a), " & ".join(b)), key=str.casefold) - return ["(%s | %s)" % (str_a, str_b)] - if not isinstance(value, str): - value = " & ".join(value) - return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold) - def handle_license_vars(srctree, lines_before, handled, extravalues, d): lichandled = [x for x in handled if x[0] == 'license'] if lichandled: From patchwork Fri Jun 20 15:14:26 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65360 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id BA40BC7115C for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2675.1750432481838287330 for ; Fri, 20 Jun 2025 08:14:41 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E5446176A; Fri, 20 Jun 2025 08:14:21 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id F0A463F673; Fri, 20 Jun 2025 08:14:40 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 09/12] classes/go-mod-update-modules: add class to generate module list Date: Fri, 20 Jun 2025 16:14:26 +0100 Message-ID: <20250620151429.3210879-9-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219136 Almost entirely based on the create_go.py module for recipetool by Christian Lindeberg , this instead has the logic inside a class that can be used to update the list of Go modules that are used, both SRC_URI and LICENSE. Integration with devtool upgrade will come shortly, but it needs a bit more work. Signed-off-by: Ross Burton --- .../go-mod-update-modules.bbclass | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 meta/classes-recipe/go-mod-update-modules.bbclass diff --git a/meta/classes-recipe/go-mod-update-modules.bbclass b/meta/classes-recipe/go-mod-update-modules.bbclass new file mode 100644 index 00000000000..6ef7ab553b0 --- /dev/null +++ b/meta/classes-recipe/go-mod-update-modules.bbclass @@ -0,0 +1,152 @@ +addtask do_update_modules after do_configure +do_update_modules[nostamp] = "1" +do_update_modules[network] = "1" + +# This class maintains two files, BPN-go-mods.inc and BPN-licenses.inc. +# +# -go-mods.inc will append SRC_URI with all of the Go modules that are +# dependencies of this recipe. +# +# -licenses.inc will append LICENSE and LIC_FILES_CHKSUM with the found licenses +# in the modules. +# +# These files are machine-generated and should not be modified. + +python do_update_modules() { + import subprocess, tempfile, json, re, urllib.parse + from oe.license import tidy_licenses + from oe.license_finder import find_licenses + + def unescape_path(path): + """Unescape capital letters using exclamation points.""" + return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path) + + def fold_uri(uri): + """Fold URI for sorting shorter module paths before longer.""" + return uri.replace(';', ' ').replace('/', '!') + + def parse_existing_licenses(): + hashes = {} + for url in d.getVar("LIC_FILES_CHKSUM").split(): + (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url) + if "spdx" in parm and parm["spdx"] != "Unknown": + hashes[parm["md5"]] = urllib.parse.unquote_plus(parm["spdx"]) + return hashes + + bpn = d.getVar("BPN") + thisdir = d.getVar("THISDIR") + s_dir = d.getVar("S") + + with tempfile.TemporaryDirectory(prefix='go-mod-') as mod_cache_dir: + notice = """ +# This file has been generated by go-mod-update-modules.bbclass +# +# Do not modify it by hand, as the contents will be replaced when +# running the update-modules task. + +""" + + env = dict(os.environ, GOMODCACHE=mod_cache_dir) + + source = d.expand("${WORKDIR}/${GO_SRCURI_DESTSUFFIX}") + output = subprocess.check_output(("go", "mod", "edit", "-json"), cwd=source, env=env, text=True) + go_mod = json.loads(output) + + output = subprocess.check_output(("go", "list", "-json=Dir,Module", "-deps", f"{go_mod['Module']['Path']}/..."), cwd=source, env=env, text=True) + + # + # Licenses + # + + # load hashes from the existing licenses.inc + extra_hashes = parse_existing_licenses() + + # The output of this isn't actually valid JSON, but a series of dicts. + # Wrap in [] and join the dicts with , + # Very frustrating that the json parser in python can't repeatedly + # parse from a stream. + pkgs = json.loads('[' + output.replace('}\n{', '},\n{') + ']') + # Collect licenses for the dependencies. + licenses = set() + lic_files_chksum = [] + lic_files = {} + + for pkg in pkgs: + mod = pkg.get('Module', None) + if not mod or mod.get('Main', False): + continue + + mod_dir = mod['Dir'] + + if mod_dir.startswith(s_dir): + continue + + path = os.path.relpath(mod_dir, mod_cache_dir) + + for license_name, license_file, license_md5 in find_licenses(mod['Dir'], d, first_only=True, extra_hashes=extra_hashes): + lic_files[os.path.join(path, license_file)] = (license_name, license_md5) + + for lic_file in lic_files: + license_name, license_md5 = lic_files[lic_file] + if license_name == "Unknown": + bb.warn(f"Unknown license: {lic_file} {license_md5}") + + licenses.add(lic_files[lic_file][0]) + lic_files_chksum.append( + f'file://pkg/mod/{lic_file};md5={license_md5};spdx={urllib.parse.quote_plus(license_name)}') + + licenses_filename = os.path.join(thisdir, f"{bpn}-licenses.inc") + with open(licenses_filename, "w") as f: + f.write(notice) + f.write(f'LICENSE += "& {" & ".join(tidy_licenses(licenses))}"\n\n') + f.write('LIC_FILES_CHKSUM += "\\\n') + for lic in sorted(lic_files_chksum, key=fold_uri): + f.write(' ' + lic + ' \\\n') + f.write('"\n') + + # + # Sources + # + + # Collect the module cache files downloaded by the go list command as + # the go list command knows best what the go list command needs and it + # needs more files in the module cache than the go install command as + # it doesn't do the dependency pruning mentioned in the Go module + # reference, https://go.dev/ref/mod, for go 1.17 or higher. + src_uris = [] + downloaddir = os.path.join(mod_cache_dir, 'cache', 'download') + for dirpath, _, filenames in os.walk(downloaddir): + # We want to process files under @v directories + path, base = os.path.split(os.path.relpath(dirpath, downloaddir)) + if base != '@v': + continue + + path = unescape_path(path) + zipver = None + for name in filenames: + ver, ext = os.path.splitext(name) + if ext == '.zip': + chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) + src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}') + zipver = ver + break + for name in filenames: + ver, ext = os.path.splitext(name) + if ext == '.mod' and ver != zipver: + chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) + src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}') + + + go_mods_filename = os.path.join(thisdir, f"{bpn}-go-mods.inc") + with open(go_mods_filename, "w") as f: + f.write(notice) + f.write('SRC_URI += "\\\n') + for uri in sorted(src_uris, key=fold_uri): + f.write(' ' + uri + ' \\\n') + f.write('"\n') + + subprocess.check_output(("go", "clean", "-modcache"), cwd=source, env=env, text=True) +} + +# This doesn't work as we need to wipe the inc files first so we don't try looking for LICENSE files that don't yet exist +# RECIPE_UPGRADE_EXTRA_TASKS += "do_update_modules" From patchwork Fri Jun 20 15:14:27 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65358 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B29BBC7EE24 for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.2677.1750432482569852114 for ; Fri, 20 Jun 2025 08:14:42 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id ACAAA16F2; Fri, 20 Jun 2025 08:14:22 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id B7F263F673; Fri, 20 Jun 2025 08:14:41 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 10/12] recipetool/create_go: proxy module fetching to go-mod-update-modules Date: Fri, 20 Jun 2025 16:14:27 +0100 Message-ID: <20250620151429.3210879-10-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219137 Now that the go-mod-update-modules class exists, this Go handler can create a stub recipe and then proxy the module handling to the class. Signed-off-by: Ross Burton --- scripts/lib/recipetool/create_go.py | 152 +++++++--------------------- 1 file changed, 36 insertions(+), 116 deletions(-) diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py index 3e9fc857842..78097495bac 100644 --- a/scripts/lib/recipetool/create_go.py +++ b/scripts/lib/recipetool/create_go.py @@ -11,17 +11,15 @@ from recipetool.create import RecipeHandler, handle_license_vars -from recipetool.create import find_licenses import bb.utils import json import logging import os import re +import subprocess import sys import tempfile -import urllib.parse -import urllib.request logger = logging.getLogger('recipetool') @@ -66,97 +64,6 @@ class GoRecipeHandler(RecipeHandler): return bindir - @staticmethod - def __unescape_path(path): - """Unescape capital letters using exclamation points.""" - return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path) - - @staticmethod - def __fold_uri(uri): - """Fold URI for sorting shorter module paths before longer.""" - return uri.replace(';', ' ').replace('/', '!') - - @staticmethod - def __go_run_cmd(cmd, cwd, d): - env = dict(os.environ, PATH=d.getVar('PATH'), GOMODCACHE=d.getVar('GOMODCACHE')) - return bb.process.run(cmd, env=env, shell=True, cwd=cwd) - - def __go_mod(self, go_mod, srctree, localfilesdir, extravalues, d): - moddir = d.getVar('GOMODCACHE') - - # List main packages and their dependencies with the go list command. - stdout, _ = self.__go_run_cmd(f"go list -json=Dir,Module -deps {go_mod['Module']['Path']}/...", srctree, d) - pkgs = json.loads('[' + stdout.replace('}\n{', '},\n{') + ']') - - # Collect licenses for the dependencies. - licenses = set() - lic_files_chksum = [] - lic_files = {} - for pkg in pkgs: - # TODO: If the package is in a subdirectory with its own license - # files then report those istead of the license files found in the - # module root directory. - mod = pkg.get('Module', None) - if not mod or mod.get('Main', False): - continue - path = os.path.relpath(mod['Dir'], moddir) - for lic in find_licenses(mod['Dir'], d): - lic_files[os.path.join(path, lic[1])] = (lic[0], lic[2]) - - for lic_file in lic_files: - licenses.add(lic_files[lic_file][0]) - lic_files_chksum.append( - f'file://pkg/mod/{lic_file};md5={lic_files[lic_file][1]}') - - # Collect the module cache files downloaded by the go list command as - # the go list command knows best what the go list command needs and it - # needs more files in the module cache than the go install command as - # it doesn't do the dependency pruning mentioned in the Go module - # reference, https://go.dev/ref/mod, for go 1.17 or higher. - src_uris = [] - downloaddir = os.path.join(moddir, 'cache', 'download') - for dirpath, _, filenames in os.walk(downloaddir): - path, base = os.path.split(os.path.relpath(dirpath, downloaddir)) - if base != '@v': - continue - path = self.__unescape_path(path) - zipver = None - for name in filenames: - ver, ext = os.path.splitext(name) - if ext == '.zip': - chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) - src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}') - zipver = ver - break - for name in filenames: - ver, ext = os.path.splitext(name) - if ext == '.mod' and ver != zipver: - chksum = bb.utils.sha256_file(os.path.join(dirpath, name)) - src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}') - - self.__go_run_cmd("go clean -modcache", srctree, d) - - licenses_basename = "{pn}-licenses.inc" - licenses_filename = os.path.join(localfilesdir, licenses_basename) - with open(licenses_filename, "w") as f: - f.write(f'GO_MOD_LICENSES = "{" & ".join(sorted(licenses))}"\n\n') - f.write('LIC_FILES_CHKSUM += "\\\n') - for lic in sorted(lic_files_chksum, key=self.__fold_uri): - f.write(' ' + lic + ' \\\n') - f.write('"\n') - - extravalues['extrafiles'][f"../{licenses_basename}"] = licenses_filename - - go_mods_basename = "{pn}-go-mods.inc" - go_mods_filename = os.path.join(localfilesdir, go_mods_basename) - with open(go_mods_filename, "w") as f: - f.write('SRC_URI += "\\\n') - for uri in sorted(src_uris, key=self.__fold_uri): - f.write(' ' + uri + ' \\\n') - f.write('"\n') - - extravalues['extrafiles'][f"../{go_mods_basename}"] = go_mods_filename - def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): @@ -167,37 +74,52 @@ class GoRecipeHandler(RecipeHandler): if not files: return False - d = bb.data.createCopy(tinfoil.config_data) go_bindir = self.__ensure_go() if not go_bindir: sys.exit(14) - d.prependVar('PATH', '%s:' % go_bindir) handled.append('buildsystem') classes.append("go-mod") - tmp_mod_dir = tempfile.mkdtemp(prefix='go-mod-') - d.setVar('GOMODCACHE', tmp_mod_dir) + # Use go-mod-update-modules to set the full SRC_URI and LICENSE + classes.append("go-mod-update-modules") + extravalues["run_task"] = "update_modules" - stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d) - go_mod = json.loads(stdout) - go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path']) + with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir: + env = dict(os.environ) + env["PATH"] += f":{go_bindir}" + env['GOMODCACHE'] = tmp_mod_dir - localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') - extravalues.setdefault('extrafiles', {}) + stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True) + go_mod = json.loads(stdout) + go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path']) - # Write the ${BPN}-licenses.inc and ${BPN}-go-mods.inc files - self.__go_mod(go_mod, srctree, localfilesdir, extravalues, d) + localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-') + extravalues.setdefault('extrafiles', {}) - # Do generic license handling - handle_license_vars(srctree, lines_before, handled, extravalues, d) - self.__rewrite_lic_vars(lines_before) + # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files + basename = "{pn}-licenses.inc" + filename = os.path.join(localfilesdir, basename) + with open(filename, "w") as f: + f.write("# FROM RECIPETOOL\n") + extravalues['extrafiles'][f"../{basename}"] = filename - self.__rewrite_src_uri(lines_before) + basename = "{pn}-go-mods.inc" + filename = os.path.join(localfilesdir, basename) + with open(filename, "w") as f: + f.write("# FROM RECIPETOOL\n") + extravalues['extrafiles'][f"../{basename}"] = filename - lines_before.append('require ${BPN}-licenses.inc') - lines_before.append('require ${BPN}-go-mods.inc') - lines_before.append(f'GO_IMPORT = "{go_import}"') + # Do generic license handling + d = bb.data.createCopy(tinfoil.config_data) + handle_license_vars(srctree, lines_before, handled, extravalues, d) + self.__rewrite_lic_vars(lines_before) + + self.__rewrite_src_uri(lines_before) + + lines_before.append('require ${BPN}-licenses.inc') + lines_before.append('require ${BPN}-go-mods.inc') + lines_before.append(f'GO_IMPORT = "{go_import}"') def __update_lines_before(self, updated, newlines, lines_before): if updated: @@ -210,10 +132,8 @@ class GoRecipeHandler(RecipeHandler): return updated def __rewrite_lic_vars(self, lines_before): - def varfunc(varname, origvalue, op, newlines): - if varname == 'LICENSE': - return ' & '.join((origvalue, '${GO_MOD_LICENSES}')), None, -1, True + import urllib.parse if varname == 'LIC_FILES_CHKSUM': new_licenses = [] licenses = origvalue.split('\\') @@ -235,7 +155,7 @@ class GoRecipeHandler(RecipeHandler): return origvalue, None, 0, True updated, newlines = bb.utils.edit_metadata( - lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc) + lines_before, ['LIC_FILES_CHKSUM'], varfunc) return self.__update_lines_before(updated, newlines, lines_before) def __rewrite_src_uri(self, lines_before): From patchwork Fri Jun 20 15:14:28 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65354 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A4676C71155 for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2676.1750432483326576394 for ; Fri, 20 Jun 2025 08:14:43 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 7D417176A; Fri, 20 Jun 2025 08:14:23 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 7EF7E3F673; Fri, 20 Jun 2025 08:14:42 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 11/12] oeqa/sefltest/devtool: improve assignment matching in _test_recipe_contents Date: Fri, 20 Jun 2025 16:14:28 +0100 Message-ID: <20250620151429.3210879-11-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219138 This function assumed that all assignments are done with just "=". However, being able to check += or ?= is also useful, so use a regex to split the line and be more flexible about what an assignment operator looks like. Signed-off-by: Ross Burton --- meta/lib/oeqa/selftest/cases/devtool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py index 0155ee62ee0..fcdf9159b54 100644 --- a/meta/lib/oeqa/selftest/cases/devtool.py +++ b/meta/lib/oeqa/selftest/cases/devtool.py @@ -154,7 +154,7 @@ class DevtoolTestCase(OESelftestTestCase): value = invalue invar = None elif '=' in line: - splitline = line.split('=', 1) + splitline = re.split(r"[?+:]*=[+]?", line, 1) var = splitline[0].rstrip() value = splitline[1].strip().strip('"') if value.endswith('\\'): From patchwork Fri Jun 20 15:14:29 2025 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 65359 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A67A8C77B7A for ; Fri, 20 Jun 2025 15:14:45 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.2677.1750432484143206348 for ; Fri, 20 Jun 2025 08:14:44 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 45C8F16F2; Fri, 20 Jun 2025 08:14:24 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 4F9693F673; Fri, 20 Jun 2025 08:14:43 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Cc: christian.lindeberg@axis.com Subject: [PATCH 12/12] oeqa/selftest/devtool: update create_go test to match the new behaviour Date: Fri, 20 Jun 2025 16:14:29 +0100 Message-ID: <20250620151429.3210879-12-ross.burton@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20250620151429.3210879-1-ross.burton@arm.com> References: <20250620151429.3210879-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 20 Jun 2025 15:14:45 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/219139 Update the test now that the recipe uses go-mod-update-modules. Signed-off-by: Ross Burton --- meta/lib/oeqa/selftest/cases/recipetool.py | 33 +++++++--------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py b/meta/lib/oeqa/selftest/cases/recipetool.py index 36557f270fb..6097865b3eb 100644 --- a/meta/lib/oeqa/selftest/cases/recipetool.py +++ b/meta/lib/oeqa/selftest/cases/recipetool.py @@ -757,13 +757,12 @@ class RecipetoolCreateTests(RecipetoolBase): def test_recipetool_create_go(self): # Basic test to check go recipe generation + self.maxDiff = None + temprecipe = os.path.join(self.tempdir, 'recipe') os.makedirs(temprecipe) recipefile = os.path.join(temprecipe, 'recipetool-go-test_git.bb') - deps_require_file = os.path.join(temprecipe, 'recipetool-go-test', 'recipetool-go-test-modules.inc') - lics_require_file = os.path.join(temprecipe, 'recipetool-go-test', 'recipetool-go-test-licenses.inc') - modules_txt_file = os.path.join(temprecipe, 'recipetool-go-test', 'modules.txt') srcuri = 'https://git.yoctoproject.org/recipetool-go-test.git' srcrev = "c3e213c01b6c1406b430df03ef0d1ae77de5d2f7" @@ -771,13 +770,11 @@ class RecipetoolCreateTests(RecipetoolBase): result = runCmd('recipetool create -o %s %s -S %s -B %s' % (temprecipe, srcuri, srcrev, srcbranch)) - self.maxDiff = None - inherits = ['go-vendor'] + inherits = ['go-mod', 'go-mod-update-modules'] checkvars = {} checkvars['GO_IMPORT'] = "git.yoctoproject.org/recipetool-go-test" - checkvars['SRC_URI'] = {'git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https', - 'file://modules.txt'} + checkvars['SRC_URI'] = {'git://${GO_IMPORT};protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}'} checkvars['LIC_FILES_CHKSUM'] = { 'file://src/${GO_IMPORT}/LICENSE;md5=4e3933dd47afbf115e484d11385fb3bd', 'file://src/${GO_IMPORT}/is/LICENSE;md5=62beaee5a116dd1e80161667b1df39ab' @@ -786,26 +783,16 @@ class RecipetoolCreateTests(RecipetoolBase): self._test_recipe_contents(recipefile, checkvars, inherits) self.assertNotIn('Traceback', result.output) + lics_require_file = os.path.join(temprecipe, 'recipetool-go-test-licenses.inc') + self.assertFileExists(lics_require_file) checkvars = {} - checkvars['VENDORED_LIC_FILES_CHKSUM'] = set( - ['file://src/${GO_IMPORT}/vendor/github.com/godbus/dbus/v5/LICENSE;md5=09042bd5c6c96a2b9e45ddf1bc517eed', - 'file://src/${GO_IMPORT}/vendor/github.com/matryer/is/LICENSE;md5=62beaee5a116dd1e80161667b1df39ab']) - self.assertTrue(os.path.isfile(lics_require_file)) + checkvars['LIC_FILES_CHKSUM'] = {'file://pkg/mod/github.com/godbus/dbus/v5@v5.1.0/LICENSE;md5=09042bd5c6c96a2b9e45ddf1bc517eed;spdx=BSD-2-Clause'} self._test_recipe_contents(lics_require_file, checkvars, []) - # make sure that dependencies don't mention local directory ./matryer/is - dependencies = \ - [ ('github.com/godbus/dbus','v5.1.0', 'github.com/godbus/dbus/v5', '/v5', ''), - ] - - src_uri = set() - for d in dependencies: - src_uri.add(self._go_urifiy(*d)) - + deps_require_file = os.path.join(temprecipe, 'recipetool-go-test-go-mods.inc') + self.assertFileExists(deps_require_file) checkvars = {} - checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri - - self.assertTrue(os.path.isfile(deps_require_file)) + checkvars['SRC_URI'] = {'gomod://github.com/godbus/dbus/v5;version=v5.1.0;sha256sum=03dfa8e71089a6f477310d15c4d3a036d82d028532881b50fee254358e782ad9'} self._test_recipe_contents(deps_require_file, checkvars, []) class RecipetoolTests(RecipetoolBase):