| Message ID | 20250610152443.2162164-3-daniel.turull@ericsson.com |
|---|---|
| State | Accepted |
| Headers | show |
| Series | Check compiled files to filter kernel CVEs | expand |
LGTM, thanks Reviewed-by: Joshua Watt <JPEWhacker@gmail.com> On Tue, Jun 10, 2025 at 9:25 AM <daniel.turull@ericsson.com> wrote: > > From: Daniel Turull <daniel.turull@ericsson.com> > > When SPDX_INCLUDE_COMPILED_SOURCES is enabled, only include the > source code files that are used during compilation. > > It uses debugsource information generated during do_package. > > This enables an external tool to use the SPDX information to disregard > vulnerabilities that are not compiled. > > As example, when used with the default config with linux-yocto, the spdx size is > reduced from 156MB to 61MB. > > Tested with bitbake world on oe-core. > > CC: Quentin Schulz <quentin.schulz@cherry.de> > CC: Joshua Watt <JPEWhacker@gmail.com> > CC: Peter Marko <peter.marko@siemens.com> > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > --- > meta/classes/create-spdx-2.2.bbclass | 9 ++++++ > meta/classes/spdx-common.bbclass | 3 ++ > meta/lib/oe/spdx30_tasks.py | 10 +++++++ > meta/lib/oe/spdx_common.py | 41 ++++++++++++++++++++++++++++ > 4 files changed, 63 insertions(+) > > diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass > index 7e8f8b9ff5..6fc60a1d97 100644 > --- a/meta/classes/create-spdx-2.2.bbclass > +++ b/meta/classes/create-spdx-2.2.bbclass > @@ -137,6 +137,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv > spdx_files = [] > > file_counter = 1 > + > + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" > + if check_compiled_sources: > + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) > + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") > for subdir, dirs, files in os.walk(topdir): > dirs[:] = [d for d in dirs if d not in ignore_dirs] > if subdir == str(topdir): > @@ -147,6 +152,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv > filename = str(filepath.relative_to(topdir)) > > if not filepath.is_symlink() and filepath.is_file(): > + # Check if file is compiled > + if check_compiled_sources: > + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): > + continue > spdx_file = oe.spdx.SPDXFile() > spdx_file.SPDXID = get_spdxid(file_counter) > for t in get_types(filepath): > diff --git a/meta/classes/spdx-common.bbclass b/meta/classes/spdx-common.bbclass > index 713a7fc651..ca0416d1c7 100644 > --- a/meta/classes/spdx-common.bbclass > +++ b/meta/classes/spdx-common.bbclass > @@ -26,6 +26,7 @@ SPDX_TOOL_VERSION ??= "1.0" > SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" > > SPDX_INCLUDE_SOURCES ??= "0" > +SPDX_INCLUDE_COMPILED_SOURCES ??= "0" > > SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" > SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs" > @@ -40,6 +41,8 @@ SPDX_MULTILIB_SSTATE_ARCHS ??= "${SSTATE_ARCHS}" > python () { > from oe.cve_check import extend_cve_status > extend_cve_status(d) > + if d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1": > + d.setVar("SPDX_INCLUDE_SOURCES", "1") > } > > def create_spdx_source_deps(d): > diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py > index 61d7ba45e3..beeafc2bb7 100644 > --- a/meta/lib/oe/spdx30_tasks.py > +++ b/meta/lib/oe/spdx30_tasks.py > @@ -156,6 +156,11 @@ def add_package_files( > bb.note(f"Skip {topdir}") > return spdx_files > > + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" > + if check_compiled_sources: > + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) > + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") > + > for subdir, dirs, files in os.walk(topdir, onerror=walk_error): > dirs[:] = [d for d in dirs if d not in ignore_dirs] > if subdir == str(topdir): > @@ -171,6 +176,11 @@ def add_package_files( > filename = str(filepath.relative_to(topdir)) > file_purposes = get_purposes(filepath) > > + # Check if file is compiled > + if check_compiled_sources: > + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): > + continue > + > spdx_file = objset.new_file( > get_spdxid(file_counter), > filename, > diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py > index 4caefc7673..c2dec65563 100644 > --- a/meta/lib/oe/spdx_common.py > +++ b/meta/lib/oe/spdx_common.py > @@ -242,3 +242,44 @@ def fetch_data_to_uri(fd, name): > uri = uri + "@" + fd.revision > > return uri > + > +def is_compiled_source (filename, compiled_sources, types): > + """ > + Check if the file is a compiled file > + """ > + import os > + # If we don't have compiled source, we assume all are compiled. > + if not compiled_sources: > + return True > + > + # We return always true if the file type is not in the list of compiled files. > + # Some files in the source directory are not compiled, for example, Makefiles, > + # but also python .py file. We need to include them in the SPDX. > + basename = os.path.basename(filename) > + ext = basename.partition(".")[2] > + if ext not in types: > + return True > + # Check that the file is in the list > + return filename in compiled_sources > + > +def get_compiled_sources(d): > + """ > + Get list of compiled sources from debug information and normalize the paths > + """ > + import itertools > + source_info = oe.package.read_debugsources_info(d) > + if not source_info: > + bb.debug(1, "Do not have debugsources.list. Skipping") > + return [], [] > + > + # Sources are not split now in SPDX, so we aggregate them > + sources = set(itertools.chain.from_iterable(source_info.values())) > + # Check extensions of files > + types = set() > + for src in sources: > + basename = os.path.basename(src) > + ext = basename.partition(".")[2] > + if ext not in types and ext: > + types.add(ext) > + bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {str(types)}") > + return sources, types
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass index 7e8f8b9ff5..6fc60a1d97 100644 --- a/meta/classes/create-spdx-2.2.bbclass +++ b/meta/classes/create-spdx-2.2.bbclass @@ -137,6 +137,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv spdx_files = [] file_counter = 1 + + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" + if check_compiled_sources: + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") for subdir, dirs, files in os.walk(topdir): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir): @@ -147,6 +152,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv filename = str(filepath.relative_to(topdir)) if not filepath.is_symlink() and filepath.is_file(): + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): + continue spdx_file = oe.spdx.SPDXFile() spdx_file.SPDXID = get_spdxid(file_counter) for t in get_types(filepath): diff --git a/meta/classes/spdx-common.bbclass b/meta/classes/spdx-common.bbclass index 713a7fc651..ca0416d1c7 100644 --- a/meta/classes/spdx-common.bbclass +++ b/meta/classes/spdx-common.bbclass @@ -26,6 +26,7 @@ SPDX_TOOL_VERSION ??= "1.0" SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" SPDX_INCLUDE_SOURCES ??= "0" +SPDX_INCLUDE_COMPILED_SOURCES ??= "0" SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs" @@ -40,6 +41,8 @@ SPDX_MULTILIB_SSTATE_ARCHS ??= "${SSTATE_ARCHS}" python () { from oe.cve_check import extend_cve_status extend_cve_status(d) + if d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1": + d.setVar("SPDX_INCLUDE_SOURCES", "1") } def create_spdx_source_deps(d): diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py index 61d7ba45e3..beeafc2bb7 100644 --- a/meta/lib/oe/spdx30_tasks.py +++ b/meta/lib/oe/spdx30_tasks.py @@ -156,6 +156,11 @@ def add_package_files( bb.note(f"Skip {topdir}") return spdx_files + check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1" + if check_compiled_sources: + compiled_sources, types = oe.spdx_common.get_compiled_sources(d) + bb.debug(1, f"Total compiled files: {len(compiled_sources)}") + for subdir, dirs, files in os.walk(topdir, onerror=walk_error): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir): @@ -171,6 +176,11 @@ def add_package_files( filename = str(filepath.relative_to(topdir)) file_purposes = get_purposes(filepath) + # Check if file is compiled + if check_compiled_sources: + if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types): + continue + spdx_file = objset.new_file( get_spdxid(file_counter), filename, diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py index 4caefc7673..c2dec65563 100644 --- a/meta/lib/oe/spdx_common.py +++ b/meta/lib/oe/spdx_common.py @@ -242,3 +242,44 @@ def fetch_data_to_uri(fd, name): uri = uri + "@" + fd.revision return uri + +def is_compiled_source (filename, compiled_sources, types): + """ + Check if the file is a compiled file + """ + import os + # If we don't have compiled source, we assume all are compiled. + if not compiled_sources: + return True + + # We return always true if the file type is not in the list of compiled files. + # Some files in the source directory are not compiled, for example, Makefiles, + # but also python .py file. We need to include them in the SPDX. + basename = os.path.basename(filename) + ext = basename.partition(".")[2] + if ext not in types: + return True + # Check that the file is in the list + return filename in compiled_sources + +def get_compiled_sources(d): + """ + Get list of compiled sources from debug information and normalize the paths + """ + import itertools + source_info = oe.package.read_debugsources_info(d) + if not source_info: + bb.debug(1, "Do not have debugsources.list. Skipping") + return [], [] + + # Sources are not split now in SPDX, so we aggregate them + sources = set(itertools.chain.from_iterable(source_info.values())) + # Check extensions of files + types = set() + for src in sources: + basename = os.path.basename(src) + ext = basename.partition(".")[2] + if ext not in types and ext: + types.add(ext) + bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {str(types)}") + return sources, types