@@ -1088,37 +1088,33 @@ def copydebugsources(debugsrcdir, sources, d):
os.rmdir(p)
@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
-def save_debugsources_info(debugsrcdir, sources_raw, d):
+def save_debugsources_info(sources_raw, d):
import json
import bb.compress.zstd
- if debugsrcdir and sources_raw:
- debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
- debugsources_dir = os.path.dirname(debugsources_file)
- if not os.path.isdir(debugsources_dir):
- bb.utils.mkdirhier(debugsources_dir)
- bb.utils.remove(debugsources_file)
- workdir = d.getVar("WORKDIR")
- pn = d.getVar('PN')
-
- # Kernel sources are in a different directory and are special case
- # we format the sources as expected by spdx by replacing /usr/src/kernel/
- # into BP/
- kernel_src = d.getVar('KERNEL_SRC_PATH')
- bp = d.getVar('BP')
- sources_dict = {}
- for file, src_files in sources_raw:
- file_clean = file.replace(f"{workdir}/package/","")
- sources_clean = [
- src.replace(f"{debugsrcdir}/{pn}/", "")
- if not kernel_src else src.replace(f"{kernel_src}/", f"{bp}/")
- for src in src_files
- if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/")
- ]
- sources_dict[file_clean] = sorted(sources_clean)
- num_threads = int(d.getVar("BB_NUMBER_THREADS"))
- with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
- json.dump(sources_dict, f, sort_keys=True)
+ if not sources_raw:
+ return
+
+ debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
+ debugsources_dir = os.path.dirname(debugsources_file)
+ if not os.path.isdir(debugsources_dir):
+ bb.utils.mkdirhier(debugsources_dir)
+ bb.utils.remove(debugsources_file)
+
+ workdir = d.getVar("WORKDIR")
+
+ def _filter_src_file(src):
+ if src.endswith("/"):
+ return False
+ return not any(keyword in src for keyword in ("<internal>", "<built-in>"))
+
+ sources_dict = {}
+ for file, src_files in sources_raw:
+ file_clean = file.replace(f"{workdir}/package/", "")
+ sources_dict[file_clean] = sorted(filter(_filter_src_file, src_files))
+ num_threads = int(d.getVar("BB_NUMBER_THREADS"))
+ with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
+ json.dump(sources_dict, f, sort_keys=True)
@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
def read_debugsources_info(d):
@@ -1364,7 +1360,7 @@ def process_split_and_strip_files(d):
copydebugsources(dv["srcdir"], sources, d)
# Save source info to be accessible to other tasks
- save_debugsources_info(dv["srcdir"], results, d)
+ save_debugsources_info(results, d)
#
# End of debug splitting
#
@@ -277,14 +277,44 @@ def get_compiled_sources(d):
bb.debug(1, "Do not have debugsources.list. Skipping")
return [], []
- # Sources are not split now in SPDX, so we aggregate them
- sources = set(itertools.chain.from_iterable(source_info.values()))
- # Check extensions of files
+ unpackdir = d.getVar("UNPACKDIR")
+ srcdir = d.getVar("S")
+ bp = d.getVar("BP")
+ kernel_src = d.getVar("KERNEL_SRC_PATH")
+ dbgsrc_dir = d.getVar("TARGET_DBGSRC_DIR")
+
+ # Compute the relative path of source directory from ${UNPACKDIR}.
+ # The goal is to replace ${TARGET_DBGSRC_DIR} by this relative path.
+ srcdir_rel = None
+ if srcdir and unpackdir:
+ srcdir_rel = os.path.relpath(srcdir, unpackdir)
+ if srcdir_rel.startswith(".."):
+ srcdir_rel = None
+
+ sources = set()
types = set()
- for src in sources:
+
+ # Sources are not split now in SPDX, so we aggregate them
+ for src in set(itertools.chain.from_iterable(source_info.values())):
+ # In the common case, the sources are located in ${S}. To format them as
+ # expected by SPDX, we replace /usr/src/debug/${PN}/${PV} with the path
+ # of ${S} relative to ${UNPACKDIR}.
+ if dbgsrc_dir and srcdir_rel:
+ src = src.replace(f"{dbgsrc_dir}/", f"{srcdir_rel}/")
+
+ # Kernel sources are in a different directory and are special case
+ # we format the sources as expected by spdx by replacing /usr/src/kernel/
+ # into ${BP}/
+ if kernel_src and bp:
+ src = src.replace(f"{kernel_src}/", f"{bp}/")
+
+ sources.add(src)
+
+ # Check extensions of files
basename = os.path.basename(src)
ext = basename.partition(".")[2]
- if ext not in types and ext:
+ if ext:
types.add(ext)
- bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {str(types)}")
+
+ bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {types!s}")
return sources, types
The source files saved in `${PN}-debugsources.json.zstd` are now saved as is: "/usr/src/kernel/" is no longer replaced by "${BP}/". However, the file list is still filtered to exclude "<internal>" or "<built-in>", ... With this modification, the content of `${PN}-debugsources.json.zstd` is no longer specific or tied to SPDX generation. The logic that modified the source file paths to match what SPDX generation expects is now in `oe.spdx_common.get_compiled_sources()`. Furthermore, previously, for all non-kernel recipes the source paths were never "resolved", since the KERNEL_SRC_PATH variable is always defined. So the returned source file paths always started with `/usr/src/debug/${PN}/${PV}` (the value of TARGET_DBGSRC_DIR). Update the source file paths to match what the SPDX generation expects: - In `get_patched_src()`, the sources of the recipe are extracted (again). The unpack task runs in a modified local context with UNPACKDIR set to the value of `${SPDXWORK}`, so the sources end up extracted in a sub-directory of `${SPDXWORK}`. - In `add_package_files()`, all files below topdir, set to `${SPDXWORK}`, are (recursively) listed. For each source file, if its path (relative to topdir) is in the list returned by `get_compiled_sources()`, the file is added to the SPDX SBoM. Handle this by replacing ${TARGET_DBGSRC_DIR} with the relative path of ${S} relative to ${UNPACKDIR}. If ${S} is not relative to ${UNPACKDIR}, do nothing. Signed-off-by: Benjamin Robin <benjamin.robin@bootlin.com> --- The overall goal of this series is to fix SPDX SBoM generation in such a way that it correctly includes compiled sources for "normal" recipes (a recipe which does not inherit the kernel class). This was tested with the busybox recipe. The following command list the effective list of compiled sources. python3 -m json.tool \ tmp/deploy/spdx/3.0.1/cortexa15t2hf-neon/builds/build-busybox.spdx.json | \ grep -F '.c"' This is the same list (however without the same prefix) as the extracted sources from the debug symbol: zstdcat tmp/pkgdata/qemuarm/debugsources/busybox-debugsources.json.zstd | \ python3 -m json.tool --- Changes in v2: - No longer modify the source file paths in debugsources.json.zstd - Move the logic in oe.spdx_common.get_compiled_sources() - Link to v1: https://patch.msgid.link/20260810-fix-save-debugsources-info-v1-0-2e83131bcf01@bootlin.com --- meta/lib/oe/package.py | 54 +++++++++++++++++++++------------------------- meta/lib/oe/spdx_common.py | 42 ++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 35 deletions(-) --- base-commit: 7944665886cb0bf881807a5d849ebdf42a33d1bf change-id: 20260810-fix-save-debugsources-info-b8d0ac4ea206 Best regards, -- Benjamin Robin <benjamin.robin@bootlin.com>