| Message ID | 20250619084736.1940747-1-daniel.turull@ericsson.com |
|---|---|
| State | Accepted |
| Delegated to: | Steve Sakoman |
| Headers | show |
| Series | [scarthgap,1/2] package: export debugsources in PKGDESTWORK as json | expand |
Hi Daniel, Since this is a feature addition the TSC will need to approve it before I can take this backport. I've copied the TSC so they can add this to the agenda for their next meeting. Thanks, Steve On Thu, Jun 19, 2025 at 1:47 AM Daniel Turull via lists.openembedded.org <daniel.turull=ericsson.com@lists.openembedded.org> wrote: > > From: Daniel Turull <daniel.turull@ericsson.com> > > The source information used during packaging can be use from other tasks to > have more detailed information on the files used during the compilation and > improve SPDX accuracy. > > Source files used during compilation are store as compressed zstd json in > pkgdata/debugsources/$PN-debugsources.json.zstd > Format: > { binary1: [src1, src2, ...], binary2: [src1, src2, ...] } > > I checked the sstate size, and it slightly increases using core-image-full-cmdline: > without patch: 2456792 KB sstate-cache/ > with patch: 2460028 KB sstate-cache/ > (4236 KB or 0.17%) > > (From OE-Core rev: c507dcb8a8780a42bfe68b1ebaff0909b4236e6b) > Adaptations to match spdx in scarthgap: change BP to PF > > CC: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com> > CC: Richard Purdie <richard.purdie@linuxfoundation.org> > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > --- > meta/conf/bitbake.conf | 2 ++ > meta/lib/oe/package.py | 46 ++++++++++++++++++++++++++++++++++++++++++ > 2 files changed, 48 insertions(+) > > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf > index 78f15b76ae..acf4e2d153 100644 > --- a/meta/conf/bitbake.conf > +++ b/meta/conf/bitbake.conf > @@ -989,5 +989,7 @@ oe.sstatesig.find_sstate_manifest[vardepsexclude] = "BBEXTENDCURR BBEXTENDVARIAN > oe.utils.get_multilib_datastore[vardepsexclude] = "DEFAULTTUNE_MULTILIB_ORIGINAL OVERRIDES" > oe.path.format_display[vardepsexclude] = "TOPDIR" > oe.utils.get_bb_number_threads[vardepsexclude] = "BB_NUMBER_THREADS" > +oe.package.save_debugsources_info[vardepsexclude] = "BB_NUMBER_THREADS" > +oe.package.read_debugsources_info[vardepsexclude] = "BB_NUMBER_THREADS" > oe.packagedata.emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" > oe.packagedata.read_subpkgdata_extended[vardepsexclude] = "BB_NUMBER_THREADS" > diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py > index af0923a63f..ba0d326781 100644 > --- a/meta/lib/oe/package.py > +++ b/meta/lib/oe/package.py > @@ -1038,6 +1038,49 @@ def copydebugsources(debugsrcdir, sources, d): > if os.path.exists(p) and not os.listdir(p): > os.rmdir(p) > > +def save_debugsources_info(debugsrcdir, sources_raw, d): > + import json > + import bb.compress.zstd > + if debugsrcdir and sources_raw: > + debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") > + debugsources_dir = os.path.dirname(debugsources_file) > + if not os.path.isdir(debugsources_dir): > + bb.utils.mkdirhier(debugsources_dir) > + bb.utils.remove(debugsources_file) > + > + workdir = d.getVar("WORKDIR") > + pn = d.getVar('PN') > + > + # Kernel sources are in a different directory and are special case > + # we format the sources as expected by spdx by replacing /usr/src/kernel/ > + # into BP/ > + kernel_src = d.getVar('KERNEL_SRC_PATH') > + pf = d.getVar('PF') > + sources_dict = {} > + for file, src_files in sources_raw: > + file_clean = file.replace(f"{workdir}/package/","") > + sources_clean = [ > + src.replace(f"{debugsrcdir}/{pn}/", "") > + if not kernel_src else src.replace(f"{kernel_src}/", f"{pf}/") > + for src in src_files > + if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/") > + ] > + sources_dict[file_clean] = sorted(sources_clean) > + num_threads = int(d.getVar("BB_NUMBER_THREADS")) > + with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f: > + json.dump(sources_dict, f, sort_keys=True) > + > +def read_debugsources_info(d): > + import json > + import bb.compress.zstd > + try: > + fn = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") > + num_threads = int(d.getVar("BB_NUMBER_THREADS")) > + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: > + return json.load(f) > + except FileNotFoundError: > + bb.debug(1, f"File not found: {fn}") > + return None > > def process_split_and_strip_files(d): > cpath = oe.cachedpath.CachedPath() > @@ -1269,6 +1312,9 @@ def process_split_and_strip_files(d): > # Process the dv["srcdir"] if requested... > # This copies and places the referenced sources for later debugging... > copydebugsources(dv["srcdir"], sources, d) > + > + # Save source info to be accessible to other tasks > + save_debugsources_info(dv["srcdir"], results, d) > # > # End of debug splitting > # > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#219060): https://lists.openembedded.org/g/openembedded-core/message/219060 > Mute This Topic: https://lists.openembedded.org/mt/113722386/3620601 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [steve@sakoman.com] > -=-=-=-=-=-=-=-=-=-=-=- >
Thanks Steve for letting me know. I understand. Please let me know if you need anything from me. Best regards, Daniel > -----Original Message----- > From: Steve Sakoman <steve@sakoman.com> > Sent: Thursday, 19 June 2025 17:24 > To: Daniel Turull <daniel.turull@ericsson.com> > Cc: openembedded-core@lists.openembedded.org; Mathieu Dubois-Briand > <mathieu.dubois-briand@bootlin.com>; Richard Purdie > <richard.purdie@linuxfoundation.org>; Yocto TSC <tsc@lists.yoctoproject.org> > Subject: Re: [OE-core] [scarthgap][PATCH 1/2] package: export debugsources in > PKGDESTWORK as json > > Hi Daniel, > > Since this is a feature addition the TSC will need to approve it before I can take > this backport. > > I've copied the TSC so they can add this to the agenda for their next meeting. > > Thanks, > > Steve > > On Thu, Jun 19, 2025 at 1:47 AM Daniel Turull via lists.openembedded.org > <daniel.turull=ericsson.com@lists.openembedded.org> wrote: > > > > From: Daniel Turull <daniel.turull@ericsson.com> > > > > The source information used during packaging can be use from other > > tasks to have more detailed information on the files used during the > > compilation and improve SPDX accuracy. > > > > Source files used during compilation are store as compressed zstd json > > in pkgdata/debugsources/$PN-debugsources.json.zstd > > Format: > > { binary1: [src1, src2, ...], binary2: [src1, src2, ...] } > > > > I checked the sstate size, and it slightly increases using core-image-full-cmdline: > > without patch: 2456792 KB sstate-cache/ > > with patch: 2460028 KB sstate-cache/ > > (4236 KB or 0.17%) > > > > (From OE-Core rev: c507dcb8a8780a42bfe68b1ebaff0909b4236e6b) > > Adaptations to match spdx in scarthgap: change BP to PF > > > > CC: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com> > > CC: Richard Purdie <richard.purdie@linuxfoundation.org> > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > > --- > > meta/conf/bitbake.conf | 2 ++ > > meta/lib/oe/package.py | 46 > > ++++++++++++++++++++++++++++++++++++++++++ > > 2 files changed, 48 insertions(+) > > > > diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf index > > 78f15b76ae..acf4e2d153 100644 > > --- a/meta/conf/bitbake.conf > > +++ b/meta/conf/bitbake.conf > > @@ -989,5 +989,7 @@ oe.sstatesig.find_sstate_manifest[vardepsexclude] > > = "BBEXTENDCURR BBEXTENDVARIAN > oe.utils.get_multilib_datastore[vardepsexclude] = > "DEFAULTTUNE_MULTILIB_ORIGINAL OVERRIDES" > > oe.path.format_display[vardepsexclude] = "TOPDIR" > > oe.utils.get_bb_number_threads[vardepsexclude] = "BB_NUMBER_THREADS" > > +oe.package.save_debugsources_info[vardepsexclude] = > "BB_NUMBER_THREADS" > > +oe.package.read_debugsources_info[vardepsexclude] = > "BB_NUMBER_THREADS" > > oe.packagedata.emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" > > oe.packagedata.read_subpkgdata_extended[vardepsexclude] = > "BB_NUMBER_THREADS" > > diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index > > af0923a63f..ba0d326781 100644 > > --- a/meta/lib/oe/package.py > > +++ b/meta/lib/oe/package.py > > @@ -1038,6 +1038,49 @@ def copydebugsources(debugsrcdir, sources, d): > > if os.path.exists(p) and not os.listdir(p): > > os.rmdir(p) > > > > +def save_debugsources_info(debugsrcdir, sources_raw, d): > > + import json > > + import bb.compress.zstd > > + if debugsrcdir and sources_raw: > > + debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}- > debugsources.json.zstd") > > + debugsources_dir = os.path.dirname(debugsources_file) > > + if not os.path.isdir(debugsources_dir): > > + bb.utils.mkdirhier(debugsources_dir) > > + bb.utils.remove(debugsources_file) > > + > > + workdir = d.getVar("WORKDIR") > > + pn = d.getVar('PN') > > + > > + # Kernel sources are in a different directory and are special case > > + # we format the sources as expected by spdx by replacing /usr/src/kernel/ > > + # into BP/ > > + kernel_src = d.getVar('KERNEL_SRC_PATH') > > + pf = d.getVar('PF') > > + sources_dict = {} > > + for file, src_files in sources_raw: > > + file_clean = file.replace(f"{workdir}/package/","") > > + sources_clean = [ > > + src.replace(f"{debugsrcdir}/{pn}/", "") > > + if not kernel_src else src.replace(f"{kernel_src}/", f"{pf}/") > > + for src in src_files > > + if not any(keyword in src for keyword in ("<internal>", "<built-in>")) > and not src.endswith("/") > > + ] > > + sources_dict[file_clean] = sorted(sources_clean) > > + num_threads = int(d.getVar("BB_NUMBER_THREADS")) > > + with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", > num_threads=num_threads) as f: > > + json.dump(sources_dict, f, sort_keys=True) > > + > > +def read_debugsources_info(d): > > + import json > > + import bb.compress.zstd > > + try: > > + fn = d.expand("${PKGDESTWORK}/debugsources/${PN}- > debugsources.json.zstd") > > + num_threads = int(d.getVar("BB_NUMBER_THREADS")) > > + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", > num_threads=num_threads) as f: > > + return json.load(f) > > + except FileNotFoundError: > > + bb.debug(1, f"File not found: {fn}") > > + return None > > > > def process_split_and_strip_files(d): > > cpath = oe.cachedpath.CachedPath() @@ -1269,6 +1312,9 @@ def > > process_split_and_strip_files(d): > > # Process the dv["srcdir"] if requested... > > # This copies and places the referenced sources for later debugging... > > copydebugsources(dv["srcdir"], sources, d) > > + > > + # Save source info to be accessible to other tasks > > + save_debugsources_info(dv["srcdir"], results, d) > > # > > # End of debug splitting > > # > > > > -=-=-=-=-=-=-=-=-=-=-=- > > Links: You receive all messages sent to this group. > > View/Reply Online (#219060): > > https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Flist > > s.openembedded.org%2Fg%2Fopenembedded- > core%2Fmessage%2F219060&data=05% > > > 7C02%7Cdaniel.turull%40ericsson.com%7C91edce15ba0d48a507bd08ddaf45505 > 1 > > > %7C92e84cebfbfd47abbe52080c6b87953f%7C0%7C0%7C638859434418210300 > %7CUnk > > > nown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIl > AiOiJ > > > XaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=Nd6J > wfT8Z > > I7PrsTzldSmYXIscMdi%2Ff6NGA6mLD3eVZ0%3D&reserved=0 > > Mute This Topic: > > https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Flist > > > s.openembedded.org%2Fmt%2F113722386%2F3620601&data=05%7C02%7Cdan > iel.tu > > > rull%40ericsson.com%7C91edce15ba0d48a507bd08ddaf455051%7C92e84cebfbf > d4 > > > 7abbe52080c6b87953f%7C0%7C0%7C638859434418229205%7CUnknown%7CT > WFpbGZsb > > > 3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIj > o > > > iTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=OcMMd%2Fjagn8lY2Si% > 2Fd0DK > > NQ9EcRUo75nmXST96JFT0Y%3D&reserved=0 > > Group Owner: openembedded-core+owner@lists.openembedded.org > > Unsubscribe: > > https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Flist > > s.openembedded.org%2Fg%2Fopenembedded- > core%2Funsub&data=05%7C02%7Cdani > > > el.turull%40ericsson.com%7C91edce15ba0d48a507bd08ddaf455051%7C92e84c > eb > > > fbfd47abbe52080c6b87953f%7C0%7C0%7C638859434418241703%7CUnknown > %7CTWFp > > > bGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiI > sIk > > > FOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=OI7it65hjB%2BkO > pZboP > > tFKEe2Zv1cWNF9FWu5jUVsEJw%3D&reserved=0 [steve@sakoman.com] > > -=-=-=-=-=-=-=-=-=-=-=- > >
On Thu, 2025-06-19 at 08:23 -0700, Steve Sakoman wrote: > Since this is a feature addition the TSC will need to approve it > before I can take this backport. > > I've copied the TSC so they can add this to the agenda for their next > meeting. The TSC discussed this today. The change adds a single data file to packagedata based on existing information we already calculate during the build so the potential risk and impact on existing users is low. Whilst this is a feature change, the uses for security and handling CVEs are extremely useful so on balance the TSC thinks this can be merged. Cheers, Richard (on behalf of the YP TSC)
Thanks
On Mon, Jun 23, 2025 at 2:33 PM Richard Purdie <richard.purdie@linuxfoundation.org> wrote: > > On Thu, 2025-06-19 at 08:23 -0700, Steve Sakoman wrote: > > Since this is a feature addition the TSC will need to approve it > > before I can take this backport. > > > > I've copied the TSC so they can add this to the agenda for their next > > meeting. > > The TSC discussed this today. The change adds a single data file to > packagedata based on existing information we already calculate during > the build so the potential risk and impact on existing users is low. > > Whilst this is a feature change, the uses for security and handling > CVEs are extremely useful so on balance the TSC thinks this can be > merged. Unfortunately this patch is causing oe-selftest failures on the autobuilder, here's a sample log: https://errors.yoctoproject.org/Errors/Details/862388/ This will need to be addressed before I can merge. Thanks, Steve
Ok. I'll take a look and see what changed from scarthgap to master. I'll resend the patch when it is fixed. I'll test with the oe-selftest Daniel > -----Original Message----- > From: Steve Sakoman <steve@sakoman.com> > Sent: Wednesday, 25 June 2025 17:27 > To: Daniel Turull <daniel.turull@ericsson.com> > Cc: openembedded-core@lists.openembedded.org; Mathieu Dubois-Briand > <mathieu.dubois-briand@bootlin.com>; Yocto TSC > <tsc@lists.yoctoproject.org>; Richard Purdie > <richard.purdie@linuxfoundation.org> > Subject: Re: [OE-core] [scarthgap][PATCH 1/2] package: export debugsources > in PKGDESTWORK as json > > On Mon, Jun 23, 2025 at 2:33 PM Richard Purdie > <richard.purdie@linuxfoundation.org> wrote: > > > > On Thu, 2025-06-19 at 08:23 -0700, Steve Sakoman wrote: > > > Since this is a feature addition the TSC will need to approve it > > > before I can take this backport. > > > > > > I've copied the TSC so they can add this to the agenda for their > > > next meeting. > > > > The TSC discussed this today. The change adds a single data file to > > packagedata based on existing information we already calculate during > > the build so the potential risk and impact on existing users is low. > > > > Whilst this is a feature change, the uses for security and handling > > CVEs are extremely useful so on balance the TSC thinks this can be > > merged. > > Unfortunately this patch is causing oe-selftest failures on the autobuilder, > here's a sample log: > > https://eur02.safelinks.protection.outlook.com/?url=https%3A%2F%2Ferrors. > yoctoproject.org%2FErrors%2FDetails%2F862388%2F&data=05%7C02%7Cdan > iel.turull%40ericsson.com%7Cdefa9e6dee2e4d7b179308ddb3fcc9fd%7C92e8 > 4cebfbfd47abbe52080c6b87953f%7C0%7C0%7C638864620472989564%7CUn > known%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAw > MCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C > &sdata=yfloJxqQl8rqqBSwuFZdlk62PfAx0RD%2ByfILVfmTgI8%3D&reserved=0 > > This will need to be addressed before I can merge. > > Thanks, > > Steve
Hi Steve,
I looked at the log and it is a bit strange. The failure in the log seems to be related to sstate. From what I see in the log sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source is failing but my patch is not touching gcc neither sstate. It is only creating a new zstd file with existing data in do_package.
2025-06-24 22:32:42,764 - oe-selftest - INFO - sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source (subunit.RemotedTestCase)
2025-06-24 22:32:42,764 - oe-selftest - INFO - ... FAIL
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/sstatetests.py", line 884, in test_gcc_runtime_vs_gcc_source
self.run_test_printdiff_changerecipe("gcc-runtime", "gcc-source", "-c do_preconfigure {}".format(gcc_source_pn),
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/sstatetests.py", line 818, in run_test_printdiff_changerecipe
result_sametmp = bitbake("-S printdiff {}".format(target))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 234, in bitbake
return runCmd(cmd, ignore_status, timeout, output_log=output_log, **options)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 212, in runCmd
raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output))
AssertionError: Command 'bitbake -S printdiff gcc-runtime' returned non-zero exit status 1:
When I run the same failing test in my local machine, it passes.
oe-selftest -r sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source -j 20 -T machine -T toolchain-user -T toolchain-system
2025-06-26 09:44:35,119 - oe-selftest - INFO - Adding layer libraries:
2025-06-26 09:44:35,119 - oe-selftest - INFO - /local/x/yocto-scarthgap/meta-yocto/meta-poky/lib
2025-06-26 09:44:35,119 - oe-selftest - INFO - /local/x/yocto-scarthgap/meta/lib
2025-06-26 09:44:35,120 - oe-selftest - INFO - /repo/x/yocto-scarthgap/meta-yocto/meta-yocto-bsp/lib
2025-06-26 09:44:35,120 - oe-selftest - INFO - /repo/x/yocto-scarthgap/openembedded-core/meta-skeleton/lib
2025-06-26 09:44:35,120 - oe-selftest - INFO - /local/x/yocto-scarthgap/openembedded-core/meta-selftest/lib
2025-06-26 09:44:35,121 - oe-selftest - INFO - Checking base configuration is valid/parsable
NOTE: Starting bitbake server...
2025-06-26 09:46:05,270 - oe-selftest - INFO - sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source (subunit.RemotedTestCase)
2025-06-26 09:46:05,271 - oe-selftest - INFO - ... ok
2025-06-26 09:46:05,271 - oe-selftest - INFO - 0: 1/1 1/1 (89.19s) (0 failed) (sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source)
2025-06-26 09:46:14,303 - oe-selftest - INFO - ----------------------------------------------------------------------
2025-06-26 09:46:14,303 - oe-selftest - INFO - Ran 1 test in 98.415s
2025-06-26 09:46:14,303 - oe-selftest - INFO - OK
2025-06-26 09:46:18,464 - oe-selftest - INFO - RESULTS:
2025-06-26 09:46:18,464 - oe-selftest - INFO - RESULTS - sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source: PASSED (89.20s)
2025-06-26 09:46:18,465 - oe-selftest - INFO - SUMMARY:
2025-06-26 09:46:18,465 - oe-selftest - INFO - oe-selftest () - Ran 1 test in 98.418s
2025-06-26 09:46:18,466 - oe-selftest - INFO - oe-selftest - OK - All required tests passed (successes=1, skipped=0, failures=0, errors=0)
Is it possible that was a temporally glitch in the CI, or related to the latest update to gcc 13.4?
Best regards,
Daniel
> -----Original Message-----
> From: Steve Sakoman <steve@sakoman.com>
> Sent: Wednesday, 25 June 2025 17:27
> To: Daniel Turull <daniel.turull@ericsson.com>
> Cc: openembedded-core@lists.openembedded.org; Mathieu Dubois-Briand
> <mathieu.dubois-briand@bootlin.com>; Yocto TSC <tsc@lists.yoctoproject.org>;
> Richard Purdie <richard.purdie@linuxfoundation.org>
> Subject: Re: [OE-core] [scarthgap][PATCH 1/2] package: export debugsources in
> PKGDESTWORK as json
>
> On Mon, Jun 23, 2025 at 2:33 PM Richard Purdie
> <richard.purdie@linuxfoundation.org> wrote:
> >
> > On Thu, 2025-06-19 at 08:23 -0700, Steve Sakoman wrote:
> > > Since this is a feature addition the TSC will need to approve it
> > > before I can take this backport.
> > >
> > > I've copied the TSC so they can add this to the agenda for their
> > > next meeting.
> >
> > The TSC discussed this today. The change adds a single data file to
> > packagedata based on existing information we already calculate during
> > the build so the potential risk and impact on existing users is low.
> >
> > Whilst this is a feature change, the uses for security and handling
> > CVEs are extremely useful so on balance the TSC thinks this can be
> > merged.
>
> Unfortunately this patch is causing oe-selftest failures on the autobuilder, here's a
> sample log:
>
> https://errors.yoct/
> oproject.org%2FErrors%2FDetails%2F862388%2F&data=05%7C02%7Cdaniel.turu
> ll%40ericsson.com%7Cdefa9e6dee2e4d7b179308ddb3fcc9fd%7C92e84cebfbfd4
> 7abbe52080c6b87953f%7C0%7C0%7C638864620472989564%7CUnknown%7CT
> WFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4
> zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=yfloJxqQl8r
> qqBSwuFZdlk62PfAx0RD%2ByfILVfmTgI8%3D&reserved=0
>
> This will need to be addressed before I can merge.
>
> Thanks,
>
> Steve
Hi Daniel, On Thu, Jun 26, 2025 at 4:37 AM Daniel Turull <daniel.turull@ericsson.com> wrote: > > Hi Steve, > I looked at the log and it is a bit strange. The failure in the log seems to be related to sstate. From what I see in the log sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source is failing but my patch is not touching gcc neither sstate. It is only creating a new zstd file with existing data in do_package. > > 2025-06-24 22:32:42,764 - oe-selftest - INFO - sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source (subunit.RemotedTestCase) > 2025-06-24 22:32:42,764 - oe-selftest - INFO - ... FAIL > > File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/sstatetests.py", line 884, in test_gcc_runtime_vs_gcc_source > self.run_test_printdiff_changerecipe("gcc-runtime", "gcc-source", "-c do_preconfigure {}".format(gcc_source_pn), > File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/selftest/cases/sstatetests.py", line 818, in run_test_printdiff_changerecipe > result_sametmp = bitbake("-S printdiff {}".format(target)) > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ > File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 234, in bitbake > return runCmd(cmd, ignore_status, timeout, output_log=output_log, **options) > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ > File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/meta/lib/oeqa/utils/commands.py", line 212, in runCmd > raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output)) > AssertionError: Command 'bitbake -S printdiff gcc-runtime' returned non-zero exit status 1: > > > When I run the same failing test in my local machine, it passes. > oe-selftest -r sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source -j 20 -T machine -T toolchain-user -T toolchain-system > > 2025-06-26 09:44:35,119 - oe-selftest - INFO - Adding layer libraries: > 2025-06-26 09:44:35,119 - oe-selftest - INFO - /local/x/yocto-scarthgap/meta-yocto/meta-poky/lib > 2025-06-26 09:44:35,119 - oe-selftest - INFO - /local/x/yocto-scarthgap/meta/lib > 2025-06-26 09:44:35,120 - oe-selftest - INFO - /repo/x/yocto-scarthgap/meta-yocto/meta-yocto-bsp/lib > 2025-06-26 09:44:35,120 - oe-selftest - INFO - /repo/x/yocto-scarthgap/openembedded-core/meta-skeleton/lib > 2025-06-26 09:44:35,120 - oe-selftest - INFO - /local/x/yocto-scarthgap/openembedded-core/meta-selftest/lib > 2025-06-26 09:44:35,121 - oe-selftest - INFO - Checking base configuration is valid/parsable > NOTE: Starting bitbake server... > 2025-06-26 09:46:05,270 - oe-selftest - INFO - sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source (subunit.RemotedTestCase) > 2025-06-26 09:46:05,271 - oe-selftest - INFO - ... ok > 2025-06-26 09:46:05,271 - oe-selftest - INFO - 0: 1/1 1/1 (89.19s) (0 failed) (sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source) > 2025-06-26 09:46:14,303 - oe-selftest - INFO - ---------------------------------------------------------------------- > 2025-06-26 09:46:14,303 - oe-selftest - INFO - Ran 1 test in 98.415s > 2025-06-26 09:46:14,303 - oe-selftest - INFO - OK > 2025-06-26 09:46:18,464 - oe-selftest - INFO - RESULTS: > 2025-06-26 09:46:18,464 - oe-selftest - INFO - RESULTS - sstatetests.SStatePrintdiff.test_gcc_runtime_vs_gcc_source: PASSED (89.20s) > 2025-06-26 09:46:18,465 - oe-selftest - INFO - SUMMARY: > 2025-06-26 09:46:18,465 - oe-selftest - INFO - oe-selftest () - Ran 1 test in 98.418s > 2025-06-26 09:46:18,466 - oe-selftest - INFO - oe-selftest - OK - All required tests passed (successes=1, skipped=0, failures=0, errors=0) > > Is it possible that was a temporally glitch in the CI, or related to the latest update to gcc 13.4? It is possible! I ran the test 4 times on that first day, both with and without your patches. Failed both times with and passed both times without. And oe-selftest failed on all distros tested. So my assumption was that it was patch related. I added back your patches and tried again this morning. All oe-selftests passed! So I'm going to leave them in my test queue for a few days to see if the issue reappears. If not, I'll merge them. Steve > > -----Original Message----- > > From: Steve Sakoman <steve@sakoman.com> > > Sent: Wednesday, 25 June 2025 17:27 > > To: Daniel Turull <daniel.turull@ericsson.com> > > Cc: openembedded-core@lists.openembedded.org; Mathieu Dubois-Briand > > <mathieu.dubois-briand@bootlin.com>; Yocto TSC <tsc@lists.yoctoproject.org>; > > Richard Purdie <richard.purdie@linuxfoundation.org> > > Subject: Re: [OE-core] [scarthgap][PATCH 1/2] package: export debugsources in > > PKGDESTWORK as json > > > > On Mon, Jun 23, 2025 at 2:33 PM Richard Purdie > > <richard.purdie@linuxfoundation.org> wrote: > > > > > > On Thu, 2025-06-19 at 08:23 -0700, Steve Sakoman wrote: > > > > Since this is a feature addition the TSC will need to approve it > > > > before I can take this backport. > > > > > > > > I've copied the TSC so they can add this to the agenda for their > > > > next meeting. > > > > > > The TSC discussed this today. The change adds a single data file to > > > packagedata based on existing information we already calculate during > > > the build so the potential risk and impact on existing users is low. > > > > > > Whilst this is a feature change, the uses for security and handling > > > CVEs are extremely useful so on balance the TSC thinks this can be > > > merged. > > > > Unfortunately this patch is causing oe-selftest failures on the autobuilder, here's a > > sample log: > > > > https://errors.yoct/ > > oproject.org%2FErrors%2FDetails%2F862388%2F&data=05%7C02%7Cdaniel.turu > > ll%40ericsson.com%7Cdefa9e6dee2e4d7b179308ddb3fcc9fd%7C92e84cebfbfd4 > > 7abbe52080c6b87953f%7C0%7C0%7C638864620472989564%7CUnknown%7CT > > WFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4 > > zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=yfloJxqQl8r > > qqBSwuFZdlk62PfAx0RD%2ByfILVfmTgI8%3D&reserved=0 > > > > This will need to be addressed before I can merge. > > > > Thanks, > > > > Steve
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf index 78f15b76ae..acf4e2d153 100644 --- a/meta/conf/bitbake.conf +++ b/meta/conf/bitbake.conf @@ -989,5 +989,7 @@ oe.sstatesig.find_sstate_manifest[vardepsexclude] = "BBEXTENDCURR BBEXTENDVARIAN oe.utils.get_multilib_datastore[vardepsexclude] = "DEFAULTTUNE_MULTILIB_ORIGINAL OVERRIDES" oe.path.format_display[vardepsexclude] = "TOPDIR" oe.utils.get_bb_number_threads[vardepsexclude] = "BB_NUMBER_THREADS" +oe.package.save_debugsources_info[vardepsexclude] = "BB_NUMBER_THREADS" +oe.package.read_debugsources_info[vardepsexclude] = "BB_NUMBER_THREADS" oe.packagedata.emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" oe.packagedata.read_subpkgdata_extended[vardepsexclude] = "BB_NUMBER_THREADS" diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index af0923a63f..ba0d326781 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -1038,6 +1038,49 @@ def copydebugsources(debugsrcdir, sources, d): if os.path.exists(p) and not os.listdir(p): os.rmdir(p) +def save_debugsources_info(debugsrcdir, sources_raw, d): + import json + import bb.compress.zstd + if debugsrcdir and sources_raw: + debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + debugsources_dir = os.path.dirname(debugsources_file) + if not os.path.isdir(debugsources_dir): + bb.utils.mkdirhier(debugsources_dir) + bb.utils.remove(debugsources_file) + + workdir = d.getVar("WORKDIR") + pn = d.getVar('PN') + + # Kernel sources are in a different directory and are special case + # we format the sources as expected by spdx by replacing /usr/src/kernel/ + # into BP/ + kernel_src = d.getVar('KERNEL_SRC_PATH') + pf = d.getVar('PF') + sources_dict = {} + for file, src_files in sources_raw: + file_clean = file.replace(f"{workdir}/package/","") + sources_clean = [ + src.replace(f"{debugsrcdir}/{pn}/", "") + if not kernel_src else src.replace(f"{kernel_src}/", f"{pf}/") + for src in src_files + if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/") + ] + sources_dict[file_clean] = sorted(sources_clean) + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f: + json.dump(sources_dict, f, sort_keys=True) + +def read_debugsources_info(d): + import json + import bb.compress.zstd + try: + fn = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd") + num_threads = int(d.getVar("BB_NUMBER_THREADS")) + with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: + return json.load(f) + except FileNotFoundError: + bb.debug(1, f"File not found: {fn}") + return None def process_split_and_strip_files(d): cpath = oe.cachedpath.CachedPath() @@ -1269,6 +1312,9 @@ def process_split_and_strip_files(d): # Process the dv["srcdir"] if requested... # This copies and places the referenced sources for later debugging... copydebugsources(dv["srcdir"], sources, d) + + # Save source info to be accessible to other tasks + save_debugsources_info(dv["srcdir"], results, d) # # End of debug splitting #