| Message ID | 2cf468a3da7c7db15ee42c14404a71aaacab0f28.1780354513.git.tim.orling@konsulko.com |
|---|---|
| State | New |
| Headers | show |
| Series | [yocto-autobuilder-helper,v2,01/10] scripts/utils: fix stale extraction dir when tarball is updated | expand |
On Mon, 2026-06-01 at 16:18 -0700, Tim Orling via lists.yoctoproject.org wrote: > From: Tim Orling <tim.orling@konsulko.com> > > Introduce the vcontainer-tarball SDK plumbing used by container build > jobs. The vcontainer-tarball is a meta-virtualization-derived SDK > (modelled after buildtools-tarball) that ships the container build > toolchain so worker jobs do not need to rebuild it for every step. > > * scripts/utils.py: add setup_vcontainer_tarball(), and add an > env_glob keyword argument to setup_tools_tarball() and > enable_tools_tarball() so the vcontainer-tarball can source its > specific environment-setup-ci file rather than the universal > glob. > * scripts/run-config: source the vcontainer-tarball environment > for build-targets / cmds / test-targets / plain-cmds steps, > gated by a new NOVCONTAINER step variable so individual steps > (such as the dashboard indexing step) can opt out independently > of NOBUILDTOOLS. > * scripts/shared-repo-unpack: invoke setup_vcontainer_tarball so > workers extract the SDK during unpack. > * scripts/publish-artefacts: publish the vcontainer-tarball > artefact so downstream test jobs can fetch a stable SDK. > > AI-Generated: Claude Cowork Opus 4.7 > Signed-off-by: Tim Orling <tim.orling@konsulko.com> > --- > config.json | 1 + > scripts/publish-artefacts | 5 +++++ > scripts/run-config | 19 +++++++++++++++++++ > scripts/shared-repo-unpack | 1 + > scripts/utils.py | 22 ++++++++++++++++++---- > 5 files changed, 44 insertions(+), 4 deletions(-) > > diff --git a/config.json b/config.json > index 7df4271..16d8a04 100644 > --- a/config.json > +++ b/config.json > @@ -1433,6 +1433,7 @@ > "step3" : { > "shortname" : "Populate/update dashboard site", > "NOBUILDTOOLS" : true, > + "NOVCONTAINER" : true, > "EXTRACMDS" : ["${SCRIPTSDIR}/run-dashboard-index ${HELPERBUILDDIR}/../"] > } > }, > diff --git a/scripts/publish-artefacts b/scripts/publish-artefacts > index e56e131..0e820e9 100755 > --- a/scripts/publish-artefacts > +++ b/scripts/publish-artefacts > @@ -146,5 +146,10 @@ case "$target" in > sha256sums $TMPDIR/deploy/images/qemux86-64 > cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86-64/*qemux86* $DEST/patchtest > ;; > + "vcontainer-tarball") > + mkdir -p $DEST/vcontainer-tarball > + sha256sums $TMPDIR/deploy/sdk > + cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/*vcontainer* $DEST/vcontainer-tarball > + ;; > esac > > diff --git a/scripts/run-config b/scripts/run-config > index e896234..0f5a26a 100755 > --- a/scripts/run-config > +++ b/scripts/run-config > @@ -153,6 +153,25 @@ else: > if args.phase == "init" and args.stepname == "buildtools": > sys.exit(0) > > +if jcfg: > + vcontainer = utils.setup_vcontainer_tarball(ourconfig, args.workername, None, checkonly=True) > + if vcontainer: > + addentry("vcontainer", "Setup vcontainer tarball", "init") > +else: > + # If we're executing a specific step, check whether vcontainer is disabled for it > + vcontainer = True > + if args.stepname in ("build-targets", "cmds", "test-targets", "plain-cmds"): > + try: > + vcontainer = not utils.getconfigvar("NOVCONTAINER", ourconfig, args.target, int(args.phase)) > + except ValueError: > + # Not an integer step phase > + pass I suspect this logic also needs to be in the if jcfg block above, else it will add the task to all jobs on the autobuilder, then just do nothing in the task. If there isn't anything to do, we may as well just not add it all? Cheers, Richard
On Mon, 2026-06-01 at 16:18 -0700, Tim Orling via lists.yoctoproject.org wrote: > From: Tim Orling <tim.orling@konsulko.com> > > Introduce the vcontainer-tarball SDK plumbing used by container build > jobs. The vcontainer-tarball is a meta-virtualization-derived SDK > (modelled after buildtools-tarball) that ships the container build > toolchain so worker jobs do not need to rebuild it for every step. > > * scripts/utils.py: add setup_vcontainer_tarball(), and add an > env_glob keyword argument to setup_tools_tarball() and > enable_tools_tarball() so the vcontainer-tarball can source its > specific environment-setup-ci file rather than the universal > glob. > * scripts/run-config: source the vcontainer-tarball environment > for build-targets / cmds / test-targets / plain-cmds steps, > gated by a new NOVCONTAINER step variable so individual steps > (such as the dashboard indexing step) can opt out independently > of NOBUILDTOOLS. > * scripts/shared-repo-unpack: invoke setup_vcontainer_tarball so > workers extract the SDK during unpack. > * scripts/publish-artefacts: publish the vcontainer-tarball > artefact so downstream test jobs can fetch a stable SDK. > > AI-Generated: Claude Cowork Opus 4.7 > Signed-off-by: Tim Orling <tim.orling@konsulko.com> > --- > config.json | 1 + > scripts/publish-artefacts | 5 +++++ > scripts/run-config | 19 +++++++++++++++++++ > scripts/shared-repo-unpack | 1 + > scripts/utils.py | 22 ++++++++++++++++++---- > 5 files changed, 44 insertions(+), 4 deletions(-) > > diff --git a/config.json b/config.json > index 7df4271..16d8a04 100644 > --- a/config.json > +++ b/config.json > @@ -1433,6 +1433,7 @@ > "step3" : { > "shortname" : "Populate/update dashboard site", > "NOBUILDTOOLS" : true, > + "NOVCONTAINER" : true, > "EXTRACMDS" : ["${SCRIPTSDIR}/run-dashboard-index ${HELPERBUILDDIR}/../"] > } > }, Why is this step needing to opt out of a vcontainer? vcontainers are only enabled for specific steps and off by default? I looked further and realised you're basically using a vcontainer everywhere as far as I can tell? Why do we want to have the overhead of doing that outside of the vcontainer builds? For buildtools, it makes sense we need it everywhere except where we configure otherwise but I think for vcontainer, you want to opposite, you only use it in jobs where we actually need/use it? Cheers, Richard
On Fri, Jun 5, 2026 at 9:26 AM Richard Purdie via lists.yoctoproject.org <richard.purdie=linuxfoundation.org@lists.yoctoproject.org> wrote: > On Mon, 2026-06-01 at 16:18 -0700, Tim Orling via lists.yoctoproject.org > wrote: > > From: Tim Orling <tim.orling@konsulko.com> > > > > Introduce the vcontainer-tarball SDK plumbing used by container build > > jobs. The vcontainer-tarball is a meta-virtualization-derived SDK > > (modelled after buildtools-tarball) that ships the container build > > toolchain so worker jobs do not need to rebuild it for every step. > > > > * scripts/utils.py: add setup_vcontainer_tarball(), and add an > > env_glob keyword argument to setup_tools_tarball() and > > enable_tools_tarball() so the vcontainer-tarball can source its > > specific environment-setup-ci file rather than the universal > > glob. > > * scripts/run-config: source the vcontainer-tarball environment > > for build-targets / cmds / test-targets / plain-cmds steps, > > gated by a new NOVCONTAINER step variable so individual steps > > (such as the dashboard indexing step) can opt out independently > > of NOBUILDTOOLS. > > * scripts/shared-repo-unpack: invoke setup_vcontainer_tarball so > > workers extract the SDK during unpack. > > * scripts/publish-artefacts: publish the vcontainer-tarball > > artefact so downstream test jobs can fetch a stable SDK. > > > > AI-Generated: Claude Cowork Opus 4.7 > > Signed-off-by: Tim Orling <tim.orling@konsulko.com> > > --- > > config.json | 1 + > > scripts/publish-artefacts | 5 +++++ > > scripts/run-config | 19 +++++++++++++++++++ > > scripts/shared-repo-unpack | 1 + > > scripts/utils.py | 22 ++++++++++++++++++---- > > 5 files changed, 44 insertions(+), 4 deletions(-) > > > > diff --git a/config.json b/config.json > > index 7df4271..16d8a04 100644 > > --- a/config.json > > +++ b/config.json > > @@ -1433,6 +1433,7 @@ > > "step3" : { > > "shortname" : "Populate/update dashboard site", > > "NOBUILDTOOLS" : true, > > + "NOVCONTAINER" : true, > > "EXTRACMDS" : ["${SCRIPTSDIR}/run-dashboard-index > ${HELPERBUILDDIR}/../"] > > } > > }, > > Why is this step needing to opt out of a vcontainer? vcontainers are > only enabled for specific steps and off by default? > > I looked further and realised you're basically using a vcontainer > everywhere as far as I can tell? Why do we want to have the overhead of > doing that outside of the vcontainer builds? > > For buildtools, it makes sense we need it everywhere except where we > configure otherwise but I think for vcontainer, you want to opposite, > you only use it in jobs where we actually need/use it? > > The simple answer is that the 'buildtools' pattern was followed and an attempt was made to reuse the 'buildtools' code as much as possible. Upon further review, the 'extratools' pattern is a better fit. This was addressed in v3. Cheers, > > Richard > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#4153): > https://lists.yoctoproject.org/g/yocto-patches/message/4153 > Mute This Topic: https://lists.yoctoproject.org/mt/119603244/924729 > Group Owner: yocto-patches+owner@lists.yoctoproject.org > Unsubscribe: > https://lists.yoctoproject.org/g/yocto-patches/leave/13169857/924729/1023951714/xyzzy > [ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > > >
diff --git a/config.json b/config.json index 7df4271..16d8a04 100644 --- a/config.json +++ b/config.json @@ -1433,6 +1433,7 @@ "step3" : { "shortname" : "Populate/update dashboard site", "NOBUILDTOOLS" : true, + "NOVCONTAINER" : true, "EXTRACMDS" : ["${SCRIPTSDIR}/run-dashboard-index ${HELPERBUILDDIR}/../"] } }, diff --git a/scripts/publish-artefacts b/scripts/publish-artefacts index e56e131..0e820e9 100755 --- a/scripts/publish-artefacts +++ b/scripts/publish-artefacts @@ -146,5 +146,10 @@ case "$target" in sha256sums $TMPDIR/deploy/images/qemux86-64 cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86-64/*qemux86* $DEST/patchtest ;; + "vcontainer-tarball") + mkdir -p $DEST/vcontainer-tarball + sha256sums $TMPDIR/deploy/sdk + cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/*vcontainer* $DEST/vcontainer-tarball + ;; esac diff --git a/scripts/run-config b/scripts/run-config index e896234..0f5a26a 100755 --- a/scripts/run-config +++ b/scripts/run-config @@ -153,6 +153,25 @@ else: if args.phase == "init" and args.stepname == "buildtools": sys.exit(0) +if jcfg: + vcontainer = utils.setup_vcontainer_tarball(ourconfig, args.workername, None, checkonly=True) + if vcontainer: + addentry("vcontainer", "Setup vcontainer tarball", "init") +else: + # If we're executing a specific step, check whether vcontainer is disabled for it + vcontainer = True + if args.stepname in ("build-targets", "cmds", "test-targets", "plain-cmds"): + try: + vcontainer = not utils.getconfigvar("NOVCONTAINER", ourconfig, args.target, int(args.phase)) + except ValueError: + # Not an integer step phase + pass + + if vcontainer: + utils.setup_vcontainer_tarball(ourconfig, args.workername, args.builddir + "/../vcontainer-tarball") + if args.phase == "init" and args.stepname == "vcontainer": + sys.exit(0) + extratools = utils.getconfigvar("extratools", ourconfig, args.target) if jcfg: if extratools: diff --git a/scripts/shared-repo-unpack b/scripts/shared-repo-unpack index 797dec6..869b214 100755 --- a/scripts/shared-repo-unpack +++ b/scripts/shared-repo-unpack @@ -77,6 +77,7 @@ for repo in sorted(repos.keys()): utils.flush() utils.setup_buildtools_tarball(ourconfig, args.workername, args.abworkdir + "/buildtools") +utils.setup_vcontainer_tarball(ourconfig, args.workername, args.abworkdir + "/vcontainer-tarball") if "bitbake" not in repos: sys.exit(0) diff --git a/scripts/utils.py b/scripts/utils.py index 112ebc2..b020a7b 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -456,8 +456,8 @@ def sha256_file(filename): pass return method.hexdigest() -def enable_tools_tarball(btdir, name): - btenv = glob.glob(btdir + "/environment-setup*") +def enable_tools_tarball(btdir, name, env_glob="/environment-setup*"): + btenv = glob.glob(btdir + env_glob) print("Using %s %s" % (name, btenv)) # We either parse or wrap all our execution calls, rock and a hard place :( with open(btenv[0], "r") as f: @@ -474,6 +474,20 @@ def enable_tools_tarball(btdir, name): if line in os.environ: del os.environ[line] +def setup_vcontainer_tarball(ourconfig, workername, vcdir, checkonly=False): + vctarball = None + if "vcontainer" in ourconfig and workername: + vccfg = getconfig("vcontainer", ourconfig) + for entry in vccfg: + if fnmatch.fnmatch(workername, entry): + vctarball = vccfg[entry] + break + + if checkonly: + return vctarball + + setup_tools_tarball(ourconfig, vcdir, vctarball, name="vcontainer-tarball", env_glob="/environment-setup-ci") + def setup_buildtools_tarball(ourconfig, workername, btdir, checkonly=False): bttarball = None if "buildtools" in ourconfig and workername: @@ -488,7 +502,7 @@ def setup_buildtools_tarball(ourconfig, workername, btdir, checkonly=False): setup_tools_tarball(ourconfig, btdir, bttarball) -def setup_tools_tarball(ourconfig, btdir, bttarball, name="buildtools"): +def setup_tools_tarball(ourconfig, btdir, bttarball, name="buildtools", env_glob="/environment-setup*"): btenv = None if bttarball: @@ -548,7 +562,7 @@ def setup_tools_tarball(ourconfig, btdir, bttarball, name="buildtools"): if not os.path.exists(btdir): print("Extracting %s %s" % (name, bttarball)) subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"]) - enable_tools_tarball(btdir, name) + enable_tools_tarball(btdir, name, env_glob) def get_string_from_version(version, milestone=None, rc=None): """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,