diff mbox series

[wrynose,4/4] package: no longer modify paths in save_debugsources_info()

Message ID 20260917-backport-wrynose-v1-4-bb7d640c2719@jp.honda
State New
Delegated to: Yoann Congal
Headers show
Series spdx: fix the compiled-sources filter | expand

Commit Message

KAZUYOSHI AKIYAMA (秋山 和慶) Sept. 17, 2026, 4:24 a.m. UTC 
From: Benjamin Robin <benjamin.robin@bootlin.com>

The source files saved in `${PN}-debugsources.json.zstd` are now saved as
is: "/usr/src/kernel/" is no longer replaced by "${BP}/". However, the
file list is still filtered to exclude "<internal>" or "<built-in>", ...

With this modification, the content of `${PN}-debugsources.json.zstd`
is no longer specific or tied to SPDX generation.

The logic that modified the source file paths to match what SPDX
generation expects is now in `oe.spdx_common.get_compiled_sources()`.

Furthermore, previously, for all non-kernel recipes the source paths were
never "resolved", since the KERNEL_SRC_PATH variable is always defined.
So the returned source file paths always started with
`/usr/src/debug/${PN}/${PV}` (the value of TARGET_DBGSRC_DIR).

Update the source file paths to match what the SPDX generation expects:
- In `get_patched_src()`, the sources of the recipe are extracted
  (again). The unpack task runs in a modified local context with
  UNPACKDIR set to the value of `${SPDXWORK}`, so the sources end up
  extracted in a sub-directory of `${SPDXWORK}`.
- In `add_package_files()`, all files below topdir, set to `${SPDXWORK}`,
  are (recursively) listed. For each source file, if its path (relative
  to topdir) is in the list returned by `get_compiled_sources()`, the
  file is added to the SPDX SBoM.

Handle this by replacing ${TARGET_DBGSRC_DIR} with the relative path of
${S} relative to ${UNPACKDIR}. If ${S} is not relative to ${UNPACKDIR},
do nothing.

Signed-off-by: Benjamin Robin <benjamin.robin@bootlin.com>
Signed-off-by: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

(cherry picked from commit b567c2f0d91fbf36886d0c3a6f6ea90c07771044)
Signed-off-by: Kazuyoshi Akiyama <kazuyoshi_akiyama@jp.honda>
---
 meta/lib/oe/package.py     | 54 +++++++++++++++++++++-------------------------
 meta/lib/oe/spdx_common.py | 42 ++++++++++++++++++++++++++++++------
 2 files changed, 61 insertions(+), 35 deletions(-)
diff mbox series

Patch

diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index c375acc124..bef3c92a25 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -1060,37 +1060,33 @@  def copydebugsources(debugsrcdir, sources, d):
                 os.rmdir(p)
 
 @bb.parse.vardepsexclude("BB_NUMBER_THREADS")
-def save_debugsources_info(debugsrcdir, sources_raw, d):
+def save_debugsources_info(sources_raw, d):
     import json
     import bb.compress.zstd
-    if debugsrcdir and sources_raw:
-        debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
-        debugsources_dir = os.path.dirname(debugsources_file)
-        if not os.path.isdir(debugsources_dir):
-            bb.utils.mkdirhier(debugsources_dir)
-        bb.utils.remove(debugsources_file)
 
-        workdir = d.getVar("WORKDIR")
-        pn = d.getVar('PN')
-
-        # Kernel sources are in a different directory and are special case
-        # we format the sources as expected by spdx by replacing /usr/src/kernel/
-        # into BP/
-        kernel_src = d.getVar('KERNEL_SRC_PATH')
-        bp = d.getVar('BP')
-        sources_dict = {}
-        for file, src_files in sources_raw:
-            file_clean = file.replace(f"{workdir}/package/","")
-            sources_clean = [
-                src.replace(f"{debugsrcdir}/{pn}/", "")
-                if not kernel_src else src.replace(f"{kernel_src}/", f"{bp}/")
-                for src in src_files
-                if not any(keyword in src for keyword in ("<internal>", "<built-in>")) and not src.endswith("/")
-            ]
-            sources_dict[file_clean] = sorted(sources_clean)
-        num_threads = int(d.getVar("BB_NUMBER_THREADS"))
-        with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
-            json.dump(sources_dict, f, sort_keys=True)
+    if not sources_raw:
+        return
+
+    debugsources_file = d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
+    debugsources_dir = os.path.dirname(debugsources_file)
+    if not os.path.isdir(debugsources_dir):
+        bb.utils.mkdirhier(debugsources_dir)
+    bb.utils.remove(debugsources_file)
+
+    workdir = d.getVar("WORKDIR")
+
+    def _filter_src_file(src):
+        if src.endswith("/"):
+            return False
+        return not any(keyword in src for keyword in ("<internal>", "<built-in>"))
+
+    sources_dict = {}
+    for file, src_files in sources_raw:
+        file_clean = file.replace(f"{workdir}/package/", "")
+        sources_dict[file_clean] = sorted(filter(_filter_src_file, src_files))
+    num_threads = int(d.getVar("BB_NUMBER_THREADS"))
+    with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
+        json.dump(sources_dict, f, sort_keys=True)
 
 @bb.parse.vardepsexclude("BB_NUMBER_THREADS")
 def read_debugsources_info(d):
@@ -1337,7 +1333,7 @@  def process_split_and_strip_files(d):
         copydebugsources(dv["srcdir"], sources, d)
 
         # Save source info to be accessible to other tasks
-        save_debugsources_info(dv["srcdir"], results, d)
+        save_debugsources_info(results, d)
     #
     # End of debug splitting
     #
diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py
index e40cf36bba..f885f1832a 100644
--- a/meta/lib/oe/spdx_common.py
+++ b/meta/lib/oe/spdx_common.py
@@ -277,14 +277,44 @@  def get_compiled_sources(d):
         bb.debug(1, "Do not have debugsources.list. Skipping")
         return [], []
 
-    # Sources are not split now in SPDX, so we aggregate them
-    sources = set(itertools.chain.from_iterable(source_info.values()))
-    # Check extensions of files
+    unpackdir = d.getVar("UNPACKDIR")
+    srcdir = d.getVar("S")
+    bp = d.getVar("BP")
+    kernel_src = d.getVar("KERNEL_SRC_PATH")
+    dbgsrc_dir = d.getVar("TARGET_DBGSRC_DIR")
+
+    # Compute the relative path of source directory from ${UNPACKDIR}.
+    # The goal is to replace ${TARGET_DBGSRC_DIR} by this relative path.
+    srcdir_rel = None
+    if srcdir and unpackdir:
+        srcdir_rel = os.path.relpath(srcdir, unpackdir)
+        if srcdir_rel.startswith(".."):
+            srcdir_rel = None
+
+    sources = set()
     types = set()
-    for src in sources:
+
+    # Sources are not split now in SPDX, so we aggregate them
+    for src in set(itertools.chain.from_iterable(source_info.values())):
+        # In the common case, the sources are located in ${S}. To format them as
+        # expected by SPDX, we replace /usr/src/debug/${PN}/${PV} with the path
+        # of ${S} relative to ${UNPACKDIR}.
+        if dbgsrc_dir and srcdir_rel:
+            src = src.replace(f"{dbgsrc_dir}/", f"{srcdir_rel}/")
+
+        # Kernel sources are in a different directory and are special case
+        # we format the sources as expected by spdx by replacing /usr/src/kernel/
+        # into ${BP}/
+        if kernel_src and bp:
+            src = src.replace(f"{kernel_src}/", f"{bp}/")
+
+        sources.add(src)
+
+        # Check extensions of files
         basename = os.path.basename(src)
         ext = basename.partition(".")[2]
-        if ext not in types and ext:
+        if ext:
             types.add(ext)
-    bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {str(types)}")
+
+    bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {types!s}")
     return sources, types