From patchwork Tue May 14 16:15:19 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 43590 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id C2BF4C04FFE for ; Tue, 14 May 2024 16:15:34 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.20267.1715703325545704699 for ; Tue, 14 May 2024 09:15:25 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0EA111007 for ; Tue, 14 May 2024 09:15:50 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.oss.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CB2533F641 for ; Tue, 14 May 2024 09:15:24 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Subject: [PATCH v2 1/3] lib/oe/package-manager: allow including self in create_packages_dir Date: Tue, 14 May 2024 16:15:19 +0000 Message-Id: <20240514161521.4082896-1-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 14 May 2024 16:15:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/199265 This function is typically used to construct a limited feed for image creation, but there are other cases when you might want a limited feed and include the current recipe's packages in it. To ensure that existing behaviour is preserved, add a boolean to control this behaviour and default it to False. Signed-off-by: Ross Burton --- meta/lib/oe/package_manager/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/meta/lib/oe/package_manager/__init__.py b/meta/lib/oe/package_manager/__init__.py index 6774cdb794d..d3b23178949 100644 --- a/meta/lib/oe/package_manager/__init__.py +++ b/meta/lib/oe/package_manager/__init__.py @@ -449,7 +449,7 @@ class PackageManager(object, metaclass=ABCMeta): return res return _append(uris, base_paths) -def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencies): +def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencies, include_self=False): """ Go through our do_package_write_X dependencies and hardlink the packages we depend upon into the repo directory. This prevents us seeing other packages that may @@ -486,14 +486,17 @@ def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencie bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?") pkgdeps = set() start = [start] - seen = set(start) + if include_self: + seen = set() + else: + seen = set(start) # Support direct dependencies (do_rootfs -> do_package_write_X) # or indirect dependencies within PN (do_populate_sdk_ext -> do_rootfs -> do_package_write_X) while start: next = [] for dep2 in start: for dep in taskdepdata[dep2][3]: - if taskdepdata[dep][0] != pn: + if include_self or taskdepdata[dep][0] != pn: if "do_" + taskname in dep: pkgdeps.add(dep) elif dep not in seen: From patchwork Tue May 14 16:15:20 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 43592 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id D7C04C25B75 for ; Tue, 14 May 2024 16:15:34 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.20269.1715703325940669626 for ; Tue, 14 May 2024 09:15:26 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 90F0A1007 for ; Tue, 14 May 2024 09:15:50 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.oss.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 5AD323F641 for ; Tue, 14 May 2024 09:15:25 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Subject: [PATCH v2 2/3] selftest/classes: add localpkgfeed class Date: Tue, 14 May 2024 16:15:20 +0000 Message-Id: <20240514161521.4082896-2-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240514161521.4082896-1-ross.burton@arm.com> References: <20240514161521.4082896-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 14 May 2024 16:15:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/199266 This class can be used to construct a subset of a deployed package feed for use in tests which iterate the deploy directory, and as such a huge feed of 30K+ packages can result in very slow tests. Signed-off-by: Ross Burton --- meta-selftest/classes/localpkgfeed.bbclass | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 meta-selftest/classes/localpkgfeed.bbclass diff --git a/meta-selftest/classes/localpkgfeed.bbclass b/meta-selftest/classes/localpkgfeed.bbclass new file mode 100644 index 00000000000..b796375e55c --- /dev/null +++ b/meta-selftest/classes/localpkgfeed.bbclass @@ -0,0 +1,27 @@ +# Create a subset of the package feed that just contain the +# packages depended on by this recipe. + +LOCALPKGFEED_DIR = "${WORKDIR}/localpkgfeed" + +addtask localpkgfeed after do_build +do_localpkgfeed[cleandirs] = "${LOCALPKGFEED_DIR}" +do_localpkgfeed[nostamp] = "1" + +def get_packaging_class(d): + package_class = d.getVar("PACKAGE_CLASSES").split()[0] + return package_class.replace("package_", "") + +python () { + packaging = get_packaging_class(d) + d.setVarFlag("do_localpkgfeed", "rdeptask", "do_package_write_" + packaging) +} + +python do_localpkgfeed() { + import oe.package_manager + + packaging = get_packaging_class(d) + deploydir = d.getVar("DEPLOY_DIR_" + packaging.upper()) + task = "package_write_" + packaging + + oe.package_manager.create_packages_dir(d, d.getVar("LOCALPKGFEED_DIR"), deploydir, task, True, True) +} From patchwork Tue May 14 16:15:21 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 43591 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id C3285C41513 for ; Tue, 14 May 2024 16:15:34 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.20270.1715703326476720739 for ; Tue, 14 May 2024 09:15:26 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 208401007 for ; Tue, 14 May 2024 09:15:51 -0700 (PDT) Received: from cesw-amp-gbt-1s-m12830-04.oss.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id DE9323F641 for ; Tue, 14 May 2024 09:15:25 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Subject: [PATCH v2 3/3] oeqa/selftest/debuginfod: use localpkgfeed to speed server startup Date: Tue, 14 May 2024 16:15:21 +0000 Message-Id: <20240514161521.4082896-3-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240514161521.4082896-1-ross.burton@arm.com> References: <20240514161521.4082896-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 14 May 2024 16:15:34 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/199267 Sometimes the debuginfod selftest fails due to a timeout, because it spends too long scanning a huge deploy directory that due to what tests were ran previously can contain 30K packages. The test only needs a subset of the feed, so use the new localpkgfeed class to construct a minimal feed before running the test. [ YOCTO #14937 ] Signed-off-by: Ross Burton --- meta/lib/oeqa/selftest/cases/debuginfod.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/debuginfod.py b/meta/lib/oeqa/selftest/cases/debuginfod.py index 505b4be8373..46c0cd87bbc 100644 --- a/meta/lib/oeqa/selftest/cases/debuginfod.py +++ b/meta/lib/oeqa/selftest/cases/debuginfod.py @@ -62,7 +62,7 @@ class Debuginfod(OESelftestTestCase): raise TimeoutError("Cannot connect debuginfod, still %d scan jobs running" % latest) - def start_debuginfod(self): + def start_debuginfod(self, feed_dir): # We assume that the caller has already bitbake'd elfutils-native:do_addto_recipe_sysroot # Save some useful paths for later @@ -82,7 +82,7 @@ class Debuginfod(OESelftestTestCase): # Disable rescanning, this is a one-shot test "--rescan-time=0", "--groom-time=0", - get_bb_var("DEPLOY_DIR"), + feed_dir, ] format = get_bb_var("PACKAGE_CLASSES").split()[0] @@ -114,11 +114,12 @@ class Debuginfod(OESelftestTestCase): self.write_config(""" TMPDIR = "${TOPDIR}/tmp-debuginfod" DISTRO_FEATURES:append = " debuginfod" +INHERIT += "localpkgfeed" """) - bitbake("elfutils-native:do_addto_recipe_sysroot xz xz:do_package") + bitbake("elfutils-native:do_addto_recipe_sysroot xz xz:do_package xz:do_localpkgfeed") try: - self.start_debuginfod() + self.start_debuginfod(get_bb_var("LOCALPKGFEED_DIR", "xz")) env = os.environ.copy() env["DEBUGINFOD_URLS"] = "http://localhost:%d/" % self.port @@ -141,12 +142,13 @@ DISTRO_FEATURES:append = " debuginfod" self.write_config(""" TMPDIR = "${TOPDIR}/tmp-debuginfod" DISTRO_FEATURES:append = " debuginfod" +INHERIT += "localpkgfeed" CORE_IMAGE_EXTRA_INSTALL += "elfutils xz" """) - bitbake("core-image-minimal elfutils-native:do_addto_recipe_sysroot") + bitbake("core-image-minimal elfutils-native:do_addto_recipe_sysroot xz:do_localpkgfeed") try: - self.start_debuginfod() + self.start_debuginfod(get_bb_var("LOCALPKGFEED_DIR", "xz")) with runqemu("core-image-minimal", runqemuparams="nographic") as qemu: cmd = "DEBUGINFOD_URLS=http://%s:%d/ debuginfod-find debuginfo /usr/bin/xz" % (qemu.server_ip, self.port)