diff mbox series

[v2] package: no longer modify paths in save_debugsources_info()

Message ID 20260822-fix-save-debugsources-info-v2-1-34fe2bd460dc@bootlin.com
State Under Review
Headers show
Series [v2] package: no longer modify paths in save_debugsources_info() | expand

Commit Message

Benjamin Robin Aug. 22, 2026, 9:46 a.m. UTC
The source files saved in `${PN}-debugsources.json.zstd` are now saved as
is: "/usr/src/kernel/" is no longer replaced by "${BP}/". However, the
file list is still filtered to exclude "<internal>" or "<built-in>", ...

With this modification, the content of `${PN}-debugsources.json.zstd`
is no longer specific or tied to SPDX generation.

The logic that modified the source file paths to match what SPDX
generation expects is now in `oe.spdx_common.get_compiled_sources()`.

Furthermore, previously, for all non-kernel recipes the source paths were
never "resolved", since the KERNEL_SRC_PATH variable is always defined.
So the returned source file paths always started with
`/usr/src/debug/${PN}/${PV}` (the value of TARGET_DBGSRC_DIR).

Update the source file paths to match what the SPDX generation expects:
- In `get_patched_src()`, the sources of the recipe are extracted
  (again). The unpack task runs in a modified local context with
  UNPACKDIR set to the value of `${SPDXWORK}`, so the sources end up
  extracted in a sub-directory of `${SPDXWORK}`.
- In `add_package_files()`, all files below topdir, set to `${SPDXWORK}`,
  are (recursively) listed. For each source file, if its path (relative
  to topdir) is in the list returned by `get_compiled_sources()`, the
  file is added to the SPDX SBoM.

Handle this by replacing ${TARGET_DBGSRC_DIR} with the relative path of
${S} relative to ${UNPACKDIR}. If ${S} is not relative to ${UNPACKDIR},
do nothing.

Signed-off-by: Benjamin Robin <benjamin.robin@bootlin.com>
---
The overall goal of this series is to fix SPDX SBoM generation in such a way
that it correctly includes compiled sources for "normal" recipes (a recipe
which does not inherit the kernel class).

This was tested with the busybox recipe. The following command list the
effective list of compiled sources.

python3 -m json.tool \
  tmp/deploy/spdx/3.0.1/cortexa15t2hf-neon/builds/build-busybox.spdx.json | \
  grep -F '.c"'

This is the same list (however without the same prefix) as the extracted
sources from the debug symbol:

zstdcat tmp/pkgdata/qemuarm/debugsources/busybox-debugsources.json.zstd | \
  python3 -m json.tool
---
Changes in v2:
- No longer modify the source file paths in debugsources.json.zstd
- Move the logic in oe.spdx_common.get_compiled_sources()
- Link to v1: https://patch.msgid.link/20260810-fix-save-debugsources-info-v1-0-2e83131bcf01@bootlin.com
---
 meta/lib/oe/package.py     | 54 +++++++++++++++++++++-------------------------
 meta/lib/oe/spdx_common.py | 42 ++++++++++++++++++++++++++++++------
 2 files changed, 61 insertions(+), 35 deletions(-)


---
base-commit: 7944665886cb0bf881807a5d849ebdf42a33d1bf
change-id: 20260810-fix-save-debugsources-info-b8d0ac4ea206

Best regards,
--  
Benjamin Robin <benjamin.robin@bootlin.com>
diff mbox series

Patch

diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 4a244ec9801e..551681d4e153 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -1088,37 +1088,33 @@  def copydebugsources(debugsrcdir, sources, d):
                 os.rmdir(p)
 
 @bb.parse.vardepsexclude("BB_NUMBER_THREADS")
-def save_debugsources_info(debugsrcdir, sources_raw, d):
+def save_debugsources_info(sources_raw, d):
     import json
     import bb.compress.zstd
-    if debugsrcdir and sources_raw:
-        debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
-        debugsources_dir = os.path.dirname(debugsources_file)
-        if not os.path.isdir(debugsources_dir):
-            bb.utils.mkdirhier(debugsources_dir)
-        bb.utils.remove(debugsources_file)
 
-        workdir = d.getVar("WORKDIR")
-        pn = d.getVar('PN')
-
-        # Kernel sources are in a different directory and are special case
-        # we format the sources as expected by spdx by replacing /usr/src/kernel/
-        # into BP/
-        kernel_src = d.getVar('KERNEL_SRC_PATH')
-        bp = d.getVar('BP')
-        sources_dict = {}
-        for file, src_files in sources_raw:
-            file_clean = file.replace(f"{workdir}/package/","")
-            sources_clean = [
-                src.replace(f"{debugsrcdir}/{pn}/", "")
-                if not kernel_src else src.replace(f"{kernel_src}/", f"{bp}/")
-                for src in src_files
-                if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/")
-            ]
-            sources_dict[file_clean] = sorted(sources_clean)
-        num_threads = int(d.getVar("BB_NUMBER_THREADS"))
-        with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
-            json.dump(sources_dict, f, sort_keys=True)
+    if not sources_raw:
+        return
+
+    debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
+    debugsources_dir = os.path.dirname(debugsources_file)
+    if not os.path.isdir(debugsources_dir):
+        bb.utils.mkdirhier(debugsources_dir)
+    bb.utils.remove(debugsources_file)
+
+    workdir = d.getVar("WORKDIR")
+
+    def _filter_src_file(src):
+        if src.endswith("/"):
+            return False
+        return not any(keyword in src for keyword in ("<internal>", "<built-in>"))
+
+    sources_dict = {}
+    for file, src_files in sources_raw:
+        file_clean = file.replace(f"{workdir}/package/", "")
+        sources_dict[file_clean] = sorted(filter(_filter_src_file, src_files))
+    num_threads = int(d.getVar("BB_NUMBER_THREADS"))
+    with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
+        json.dump(sources_dict, f, sort_keys=True)
 
 @bb.parse.vardepsexclude("BB_NUMBER_THREADS")
 def read_debugsources_info(d):
@@ -1364,7 +1360,7 @@  def process_split_and_strip_files(d):
         copydebugsources(dv["srcdir"], sources, d)
 
         # Save source info to be accessible to other tasks
-        save_debugsources_info(dv["srcdir"], results, d)
+        save_debugsources_info(results, d)
     #
     # End of debug splitting
     #
diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py
index e40cf36bba18..f885f1832ae3 100644
--- a/meta/lib/oe/spdx_common.py
+++ b/meta/lib/oe/spdx_common.py
@@ -277,14 +277,44 @@  def get_compiled_sources(d):
         bb.debug(1, "Do not have debugsources.list. Skipping")
         return [], []
 
-    # Sources are not split now in SPDX, so we aggregate them
-    sources = set(itertools.chain.from_iterable(source_info.values()))
-    # Check extensions of files
+    unpackdir = d.getVar("UNPACKDIR")
+    srcdir = d.getVar("S")
+    bp = d.getVar("BP")
+    kernel_src = d.getVar("KERNEL_SRC_PATH")
+    dbgsrc_dir = d.getVar("TARGET_DBGSRC_DIR")
+
+    # Compute the relative path of source directory from ${UNPACKDIR}.
+    # The goal is to replace ${TARGET_DBGSRC_DIR} by this relative path.
+    srcdir_rel = None
+    if srcdir and unpackdir:
+        srcdir_rel = os.path.relpath(srcdir, unpackdir)
+        if srcdir_rel.startswith(".."):
+            srcdir_rel = None
+
+    sources = set()
     types = set()
-    for src in sources:
+
+    # Sources are not split now in SPDX, so we aggregate them
+    for src in set(itertools.chain.from_iterable(source_info.values())):
+        # In the common case, the sources are located in ${S}. To format them as
+        # expected by SPDX, we replace /usr/src/debug/${PN}/${PV} with the path
+        # of ${S} relative to ${UNPACKDIR}.
+        if dbgsrc_dir and srcdir_rel:
+            src = src.replace(f"{dbgsrc_dir}/", f"{srcdir_rel}/")
+
+        # Kernel sources are in a different directory and are special case
+        # we format the sources as expected by spdx by replacing /usr/src/kernel/
+        # into ${BP}/
+        if kernel_src and bp:
+            src = src.replace(f"{kernel_src}/", f"{bp}/")
+
+        sources.add(src)
+
+        # Check extensions of files
         basename = os.path.basename(src)
         ext = basename.partition(".")[2]
-        if ext not in types and ext:
+        if ext:
             types.add(ext)
-    bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {str(types)}")
+
+    bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {types!s}")
     return sources, types