| Message ID | 20250428134205.900354-7-daniel.turull@ericsson.com |
|---|---|
| State | New |
| Headers | show |
| Series | Check compiled files to filter kernel CVEs | expand |
I see the appeal here; however I've been trying to avoid too much kernel specific stuff in SPDX. Is there a way to make this more generic so it could be used by any recipe? On Mon, Apr 28, 2025 at 7:42 AM <daniel.turull@ericsson.com> wrote: > > From: Daniel Turull <daniel.turull@ericsson.com> > > When CVE_CHECK_KERNEL_CONFIG is enabled, only include the > source code (.c, .h) files that are used during compilation. > > This enables an external tool to use the SPDX information to disregard > vulnerabilities that are not compiled. > > CC: Joshua Watt <JPEWhacker@gmail.com> > CC: Peter Marko <peter.marko@siemens.com> > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > --- > meta/classes/create-spdx-2.2.bbclass | 8 +++++++ > meta/lib/oe/spdx30_tasks.py | 8 +++++++ > meta/lib/oe/spdx_common.py | 34 ++++++++++++++++++++++++++++ > 3 files changed, 50 insertions(+) > > diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass > index 7e8f8b9ff5..6bf0c70bd4 100644 > --- a/meta/classes/create-spdx-2.2.bbclass > +++ b/meta/classes/create-spdx-2.2.bbclass > @@ -137,6 +137,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv > spdx_files = [] > > file_counter = 1 > + > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > + if check_kernel_compiled: > + kernel_sources = oe.spdx_common.get_kernel_compiled_files(d) > for subdir, dirs, files in os.walk(topdir): > dirs[:] = [d for d in dirs if d not in ignore_dirs] > if subdir == str(topdir): > @@ -147,6 +151,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv > filename = str(filepath.relative_to(topdir)) > > if not filepath.is_symlink() and filepath.is_file(): > + # When creating spdx for the kernel, we only include compiled files. > + if check_kernel_compiled: > + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): > + break > spdx_file = oe.spdx.SPDXFile() > spdx_file.SPDXID = get_spdxid(file_counter) > for t in get_types(filepath): > diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py > index ba965821f8..14f26773c5 100644 > --- a/meta/lib/oe/spdx30_tasks.py > +++ b/meta/lib/oe/spdx30_tasks.py > @@ -156,6 +156,10 @@ def add_package_files( > bb.note(f"Skip {topdir}") > return spdx_files > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > + if check_kernel_compiled: > + kernel_sources = oe.spdx_common.get_kernel_compiled_files(d) > + > for subdir, dirs, files in os.walk(topdir, onerror=walk_error): > dirs[:] = [d for d in dirs if d not in ignore_dirs] > if subdir == str(topdir): > @@ -167,6 +171,10 @@ def add_package_files( > filepath = Path(subdir) / file > if filepath.is_symlink() or not filepath.is_file(): > continue > + # When creating spdx for the kernel, we only include compiled files > + if check_kernel_compiled: > + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): > + break > > filename = str(filepath.relative_to(topdir)) > file_purposes = get_purposes(filepath) > diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py > index 4caefc7673..c87e3875c7 100644 > --- a/meta/lib/oe/spdx_common.py > +++ b/meta/lib/oe/spdx_common.py > @@ -242,3 +242,37 @@ def fetch_data_to_uri(fd, name): > uri = uri + "@" + fd.revision > > return uri > + > +def is_kernel_compiled(filename, kernel_sources, d): > + """ > + Check if the file, is a kernel compiled file > + """ > + import os > + > + _, extension = os.path.splitext(filename) > + # Special case, that we need to ignore, since this is not a source file > + if filename.rfind(".mod.c") > 0: > + return True > + # We filter .c files and header files > + if extension not in [".c", ".h"]: > + return True > + # Check that the c file is in the list > + if filename in kernel_sources: > + return True > + return False > + > +def get_kernel_compiled_files(d): > + """ > + Get results from the save_compiled files and include also header files > + """ > + import json > + import os > + kfiles = [] > + with open(d.getVar('KERNEL_SRC_FILES'), 'r') as f: > + for item in json.load(f): > + kfile = os.path.basename(item['file']) > + # Return also the correspondig header file > + hfile = ".h".join(kfile.rsplit(".c", 1)) This feels a little bit wrong to me. I think a better option would be to include all headers, since we don't have a good way of knowing which are actually used. > + kfiles.append(kfile) > + kfiles.append(hfile) > + return kfiles
I think that only kernel provides file-based CVE information right now. Peter > -----Original Message----- > From: Joshua Watt <jpewhacker@gmail.com> > Sent: Monday, April 28, 2025 15:58 > To: daniel.turull@ericsson.com > Cc: openembedded-core@lists.openembedded.org; rybczynska@gmail.com; > steve@sakoman.com; Marko, Peter (FT D EU SK BFS1) > <Peter.Marko@siemens.com>; ross.burton@arm.com; skandigraun@gmail.com > Subject: Re: [PATCH v2 6/6] spdx: add option to include only compiled kernel files > > I see the appeal here; however I've been trying to avoid too much > kernel specific stuff in SPDX. Is there a way to make this more > generic so it could be used by any recipe? > > On Mon, Apr 28, 2025 at 7:42 AM <daniel.turull@ericsson.com> wrote: > > > > From: Daniel Turull <daniel.turull@ericsson.com> > > > > When CVE_CHECK_KERNEL_CONFIG is enabled, only include the > > source code (.c, .h) files that are used during compilation. > > > > This enables an external tool to use the SPDX information to disregard > > vulnerabilities that are not compiled. > > > > CC: Joshua Watt <JPEWhacker@gmail.com> > > CC: Peter Marko <peter.marko@siemens.com> > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > > --- > > meta/classes/create-spdx-2.2.bbclass | 8 +++++++ > > meta/lib/oe/spdx30_tasks.py | 8 +++++++ > > meta/lib/oe/spdx_common.py | 34 ++++++++++++++++++++++++++++ > > 3 files changed, 50 insertions(+) > > > > diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx- > 2.2.bbclass > > index 7e8f8b9ff5..6bf0c70bd4 100644 > > --- a/meta/classes/create-spdx-2.2.bbclass > > +++ b/meta/classes/create-spdx-2.2.bbclass > > @@ -137,6 +137,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > get_spdxid, get_types, *, archiv > > spdx_files = [] > > > > file_counter = 1 > > + > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > + if check_kernel_compiled: > > + kernel_sources = oe.spdx_common.get_kernel_compiled_files(d) > > for subdir, dirs, files in os.walk(topdir): > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > if subdir == str(topdir): > > @@ -147,6 +151,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > get_spdxid, get_types, *, archiv > > filename = str(filepath.relative_to(topdir)) > > > > if not filepath.is_symlink() and filepath.is_file(): > > + # When creating spdx for the kernel, we only include compiled files. > > + if check_kernel_compiled: > > + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, > d): > > + break > > spdx_file = oe.spdx.SPDXFile() > > spdx_file.SPDXID = get_spdxid(file_counter) > > for t in get_types(filepath): > > diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py > > index ba965821f8..14f26773c5 100644 > > --- a/meta/lib/oe/spdx30_tasks.py > > +++ b/meta/lib/oe/spdx30_tasks.py > > @@ -156,6 +156,10 @@ def add_package_files( > > bb.note(f"Skip {topdir}") > > return spdx_files > > > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > + if check_kernel_compiled: > > + kernel_sources = oe.spdx_common.get_kernel_compiled_files(d) > > + > > for subdir, dirs, files in os.walk(topdir, onerror=walk_error): > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > if subdir == str(topdir): > > @@ -167,6 +171,10 @@ def add_package_files( > > filepath = Path(subdir) / file > > if filepath.is_symlink() or not filepath.is_file(): > > continue > > + # When creating spdx for the kernel, we only include compiled files > > + if check_kernel_compiled: > > + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): > > + break > > > > filename = str(filepath.relative_to(topdir)) > > file_purposes = get_purposes(filepath) > > diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py > > index 4caefc7673..c87e3875c7 100644 > > --- a/meta/lib/oe/spdx_common.py > > +++ b/meta/lib/oe/spdx_common.py > > @@ -242,3 +242,37 @@ def fetch_data_to_uri(fd, name): > > uri = uri + "@" + fd.revision > > > > return uri > > + > > +def is_kernel_compiled(filename, kernel_sources, d): > > + """ > > + Check if the file, is a kernel compiled file > > + """ > > + import os > > + > > + _, extension = os.path.splitext(filename) > > + # Special case, that we need to ignore, since this is not a source file > > + if filename.rfind(".mod.c") > 0: > > + return True > > + # We filter .c files and header files > > + if extension not in [".c", ".h"]: > > + return True > > + # Check that the c file is in the list > > + if filename in kernel_sources: > > + return True > > + return False > > + > > +def get_kernel_compiled_files(d): > > + """ > > + Get results from the save_compiled files and include also header files > > + """ > > + import json > > + import os > > + kfiles = [] > > + with open(d.getVar('KERNEL_SRC_FILES'), 'r') as f: > > + for item in json.load(f): > > + kfile = os.path.basename(item['file']) > > + # Return also the correspondig header file > > + hfile = ".h".join(kfile.rsplit(".c", 1)) > > This feels a little bit wrong to me. I think a better option would be > to include all headers, since we don't have a good way of knowing > which are actually used. > > > + kfiles.append(kfile) > > + kfiles.append(hfile) > > + return kfiles
Hi, Yes, from what I saw and Peter says only the kernel has file information that can be machine readable, and we can also extract the files that we compile. I could try to normalize how the files are input into the spdx generation and then have a generic variable that any recipe can set to enable the inclusion of a subset of files. In this case, if/when more recipes provides this, spdx doesn't need to change. Best regards Daniel -----Original Message----- From: Marko, Peter <Peter.Marko@siemens.com> Sent: Monday, 28 April 2025 16:00 To: Joshua Watt <jpewhacker@gmail.com>; Daniel Turull <daniel.turull@ericsson.com> Cc: openembedded-core@lists.openembedded.org; rybczynska@gmail.com; steve@sakoman.com; ross.burton@arm.com; skandigraun@gmail.com Subject: RE: [PATCH v2 6/6] spdx: add option to include only compiled kernel files I think that only kernel provides file-based CVE information right now. Peter > -----Original Message----- > From: Joshua Watt <jpewhacker@gmail.com> > Sent: Monday, April 28, 2025 15:58 > To: daniel.turull@ericsson.com > Cc: openembedded-core@lists.openembedded.org; rybczynska@gmail.com; > steve@sakoman.com; Marko, Peter (FT D EU SK BFS1) > <Peter.Marko@siemens.com>; ross.burton@arm.com; skandigraun@gmail.com > Subject: Re: [PATCH v2 6/6] spdx: add option to include only compiled > kernel files > > I see the appeal here; however I've been trying to avoid too much > kernel specific stuff in SPDX. Is there a way to make this more > generic so it could be used by any recipe? > > On Mon, Apr 28, 2025 at 7:42 AM <daniel.turull@ericsson.com> wrote: > > > > From: Daniel Turull <daniel.turull@ericsson.com> > > > > When CVE_CHECK_KERNEL_CONFIG is enabled, only include the source > > code (.c, .h) files that are used during compilation. > > > > This enables an external tool to use the SPDX information to > > disregard vulnerabilities that are not compiled. > > > > CC: Joshua Watt <JPEWhacker@gmail.com> > > CC: Peter Marko <peter.marko@siemens.com> > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > > --- > > meta/classes/create-spdx-2.2.bbclass | 8 +++++++ > > meta/lib/oe/spdx30_tasks.py | 8 +++++++ > > meta/lib/oe/spdx_common.py | 34 ++++++++++++++++++++++++++++ > > 3 files changed, 50 insertions(+) > > > > diff --git a/meta/classes/create-spdx-2.2.bbclass > > b/meta/classes/create-spdx- > 2.2.bbclass > > index 7e8f8b9ff5..6bf0c70bd4 100644 > > --- a/meta/classes/create-spdx-2.2.bbclass > > +++ b/meta/classes/create-spdx-2.2.bbclass > > @@ -137,6 +137,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > get_spdxid, get_types, *, archiv > > spdx_files = [] > > > > file_counter = 1 > > + > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > + if check_kernel_compiled: > > + kernel_sources = > > + oe.spdx_common.get_kernel_compiled_files(d) > > for subdir, dirs, files in os.walk(topdir): > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > if subdir == str(topdir): > > @@ -147,6 +151,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > get_spdxid, get_types, *, archiv > > filename = str(filepath.relative_to(topdir)) > > > > if not filepath.is_symlink() and filepath.is_file(): > > + # When creating spdx for the kernel, we only include compiled files. > > + if check_kernel_compiled: > > + if not oe.spdx_common.is_kernel_compiled(file, > > + kernel_sources, > d): > > + break > > spdx_file = oe.spdx.SPDXFile() > > spdx_file.SPDXID = get_spdxid(file_counter) > > for t in get_types(filepath): > > diff --git a/meta/lib/oe/spdx30_tasks.py > > b/meta/lib/oe/spdx30_tasks.py index ba965821f8..14f26773c5 100644 > > --- a/meta/lib/oe/spdx30_tasks.py > > +++ b/meta/lib/oe/spdx30_tasks.py > > @@ -156,6 +156,10 @@ def add_package_files( > > bb.note(f"Skip {topdir}") > > return spdx_files > > > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > + if check_kernel_compiled: > > + kernel_sources = > > + oe.spdx_common.get_kernel_compiled_files(d) > > + > > for subdir, dirs, files in os.walk(topdir, onerror=walk_error): > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > if subdir == str(topdir): > > @@ -167,6 +171,10 @@ def add_package_files( > > filepath = Path(subdir) / file > > if filepath.is_symlink() or not filepath.is_file(): > > continue > > + # When creating spdx for the kernel, we only include compiled files > > + if check_kernel_compiled: > > + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): > > + break > > > > filename = str(filepath.relative_to(topdir)) > > file_purposes = get_purposes(filepath) diff --git > > a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py index > > 4caefc7673..c87e3875c7 100644 > > --- a/meta/lib/oe/spdx_common.py > > +++ b/meta/lib/oe/spdx_common.py > > @@ -242,3 +242,37 @@ def fetch_data_to_uri(fd, name): > > uri = uri + "@" + fd.revision > > > > return uri > > + > > +def is_kernel_compiled(filename, kernel_sources, d): > > + """ > > + Check if the file, is a kernel compiled file > > + """ > > + import os > > + > > + _, extension = os.path.splitext(filename) > > + # Special case, that we need to ignore, since this is not a source file > > + if filename.rfind(".mod.c") > 0: > > + return True > > + # We filter .c files and header files > > + if extension not in [".c", ".h"]: > > + return True > > + # Check that the c file is in the list > > + if filename in kernel_sources: > > + return True > > + return False > > + > > +def get_kernel_compiled_files(d): > > + """ > > + Get results from the save_compiled files and include also header files > > + """ > > + import json > > + import os > > + kfiles = [] > > + with open(d.getVar('KERNEL_SRC_FILES'), 'r') as f: > > + for item in json.load(f): > > + kfile = os.path.basename(item['file']) > > + # Return also the correspondig header file > > + hfile = ".h".join(kfile.rsplit(".c", 1)) > > This feels a little bit wrong to me. I think a better option would be > to include all headers, since we don't have a good way of knowing > which are actually used. > > > + kfiles.append(kfile) > > + kfiles.append(hfile) > > + return kfiles
Hi, I missed to answer your comment. Daniel -----Original Message----- From: Marko, Peter <Peter.Marko@siemens.com> Sent: Monday, 28 April 2025 16:00 To: Joshua Watt <jpewhacker@gmail.com>; Daniel Turull <daniel.turull@ericsson.com> Cc: openembedded-core@lists.openembedded.org; rybczynska@gmail.com; steve@sakoman.com; ross.burton@arm.com; skandigraun@gmail.com Subject: RE: [PATCH v2 6/6] spdx: add option to include only compiled kernel files I think that only kernel provides file-based CVE information right now. Peter > -----Original Message----- > From: Joshua Watt <jpewhacker@gmail.com> > Sent: Monday, April 28, 2025 15:58 > To: daniel.turull@ericsson.com > Cc: openembedded-core@lists.openembedded.org; rybczynska@gmail.com; > steve@sakoman.com; Marko, Peter (FT D EU SK BFS1) > <Peter.Marko@siemens.com>; ross.burton@arm.com; skandigraun@gmail.com > Subject: Re: [PATCH v2 6/6] spdx: add option to include only compiled > kernel files > > I see the appeal here; however I've been trying to avoid too much > kernel specific stuff in SPDX. Is there a way to make this more > generic so it could be used by any recipe? > > On Mon, Apr 28, 2025 at 7:42 AM <daniel.turull@ericsson.com> wrote: > > > > From: Daniel Turull <daniel.turull@ericsson.com> > > > > When CVE_CHECK_KERNEL_CONFIG is enabled, only include the source > > code (.c, .h) files that are used during compilation. > > > > This enables an external tool to use the SPDX information to > > disregard vulnerabilities that are not compiled. > > > > CC: Joshua Watt <JPEWhacker@gmail.com> > > CC: Peter Marko <peter.marko@siemens.com> > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > > --- > > meta/classes/create-spdx-2.2.bbclass | 8 +++++++ > > meta/lib/oe/spdx30_tasks.py | 8 +++++++ > > meta/lib/oe/spdx_common.py | 34 ++++++++++++++++++++++++++++ > > 3 files changed, 50 insertions(+) > > > > diff --git a/meta/classes/create-spdx-2.2.bbclass > > b/meta/classes/create-spdx- > 2.2.bbclass > > index 7e8f8b9ff5..6bf0c70bd4 100644 > > --- a/meta/classes/create-spdx-2.2.bbclass > > +++ b/meta/classes/create-spdx-2.2.bbclass > > @@ -137,6 +137,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > get_spdxid, get_types, *, archiv > > spdx_files = [] > > > > file_counter = 1 > > + > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > + if check_kernel_compiled: > > + kernel_sources = > > + oe.spdx_common.get_kernel_compiled_files(d) > > for subdir, dirs, files in os.walk(topdir): > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > if subdir == str(topdir): > > @@ -147,6 +151,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > get_spdxid, get_types, *, archiv > > filename = str(filepath.relative_to(topdir)) > > > > if not filepath.is_symlink() and filepath.is_file(): > > + # When creating spdx for the kernel, we only include compiled files. > > + if check_kernel_compiled: > > + if not oe.spdx_common.is_kernel_compiled(file, > > + kernel_sources, > d): > > + break > > spdx_file = oe.spdx.SPDXFile() > > spdx_file.SPDXID = get_spdxid(file_counter) > > for t in get_types(filepath): > > diff --git a/meta/lib/oe/spdx30_tasks.py > > b/meta/lib/oe/spdx30_tasks.py index ba965821f8..14f26773c5 100644 > > --- a/meta/lib/oe/spdx30_tasks.py > > +++ b/meta/lib/oe/spdx30_tasks.py > > @@ -156,6 +156,10 @@ def add_package_files( > > bb.note(f"Skip {topdir}") > > return spdx_files > > > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > + if check_kernel_compiled: > > + kernel_sources = > > + oe.spdx_common.get_kernel_compiled_files(d) > > + > > for subdir, dirs, files in os.walk(topdir, onerror=walk_error): > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > if subdir == str(topdir): > > @@ -167,6 +171,10 @@ def add_package_files( > > filepath = Path(subdir) / file > > if filepath.is_symlink() or not filepath.is_file(): > > continue > > + # When creating spdx for the kernel, we only include compiled files > > + if check_kernel_compiled: > > + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): > > + break > > > > filename = str(filepath.relative_to(topdir)) > > file_purposes = get_purposes(filepath) diff --git > > a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py index > > 4caefc7673..c87e3875c7 100644 > > --- a/meta/lib/oe/spdx_common.py > > +++ b/meta/lib/oe/spdx_common.py > > @@ -242,3 +242,37 @@ def fetch_data_to_uri(fd, name): > > uri = uri + "@" + fd.revision > > > > return uri > > + > > +def is_kernel_compiled(filename, kernel_sources, d): > > + """ > > + Check if the file, is a kernel compiled file > > + """ > > + import os > > + > > + _, extension = os.path.splitext(filename) > > + # Special case, that we need to ignore, since this is not a source file > > + if filename.rfind(".mod.c") > 0: > > + return True > > + # We filter .c files and header files > > + if extension not in [".c", ".h"]: > > + return True > > + # Check that the c file is in the list > > + if filename in kernel_sources: > > + return True > > + return False > > + > > +def get_kernel_compiled_files(d): > > + """ > > + Get results from the save_compiled files and include also header files > > + """ > > + import json > > + import os > > + kfiles = [] > > + with open(d.getVar('KERNEL_SRC_FILES'), 'r') as f: > > + for item in json.load(f): > > + kfile = os.path.basename(item['file']) > > + # Return also the correspondig header file > > + hfile = ".h".join(kfile.rsplit(".c", 1)) > > This feels a little bit wrong to me. I think a better option would be > to include all headers, since we don't have a good way of knowing > which are actually used. Yes, I had my doubt regarding this. It probably doesn't have an impact in the rest of the functionality for the CVEs I'll keep only the "c" files and try to make it more generic. > > > + kfiles.append(kfile) > > + kfiles.append(hfile) > > + return kfiles
On Mon, Apr 28, 2025 at 8:12 AM Daniel Turull <daniel.turull@ericsson.com> wrote: > > Hi, > > Yes, from what I saw and Peter says only the kernel has file information that can be machine readable, > and we can also extract the files that we compile. > > I could try to normalize how the files are input into the spdx generation and then have a generic variable that any recipe can set > to enable the inclusion of a subset of files. In this case, if/when more recipes provides this, > spdx doesn't need to change. I was thinking that maybe the way to do this is to have this somehow call a function when enabled, and then have the recipe (or even better the build system bbclass) implement that function; that way each recipe or build system can implement the correct method for collecting the data. I'm not quite sure if that's the best way to do it though > > Best regards > Daniel > > -----Original Message----- > From: Marko, Peter <Peter.Marko@siemens.com> > Sent: Monday, 28 April 2025 16:00 > To: Joshua Watt <jpewhacker@gmail.com>; Daniel Turull <daniel.turull@ericsson.com> > Cc: openembedded-core@lists.openembedded.org; rybczynska@gmail.com; steve@sakoman.com; ross.burton@arm.com; skandigraun@gmail.com > Subject: RE: [PATCH v2 6/6] spdx: add option to include only compiled kernel files > > I think that only kernel provides file-based CVE information right now. > > Peter > > > -----Original Message----- > > From: Joshua Watt <jpewhacker@gmail.com> > > Sent: Monday, April 28, 2025 15:58 > > To: daniel.turull@ericsson.com > > Cc: openembedded-core@lists.openembedded.org; rybczynska@gmail.com; > > steve@sakoman.com; Marko, Peter (FT D EU SK BFS1) > > <Peter.Marko@siemens.com>; ross.burton@arm.com; skandigraun@gmail.com > > Subject: Re: [PATCH v2 6/6] spdx: add option to include only compiled > > kernel files > > > > I see the appeal here; however I've been trying to avoid too much > > kernel specific stuff in SPDX. Is there a way to make this more > > generic so it could be used by any recipe? > > > > On Mon, Apr 28, 2025 at 7:42 AM <daniel.turull@ericsson.com> wrote: > > > > > > From: Daniel Turull <daniel.turull@ericsson.com> > > > > > > When CVE_CHECK_KERNEL_CONFIG is enabled, only include the source > > > code (.c, .h) files that are used during compilation. > > > > > > This enables an external tool to use the SPDX information to > > > disregard vulnerabilities that are not compiled. > > > > > > CC: Joshua Watt <JPEWhacker@gmail.com> > > > CC: Peter Marko <peter.marko@siemens.com> > > > Signed-off-by: Daniel Turull <daniel.turull@ericsson.com> > > > --- > > > meta/classes/create-spdx-2.2.bbclass | 8 +++++++ > > > meta/lib/oe/spdx30_tasks.py | 8 +++++++ > > > meta/lib/oe/spdx_common.py | 34 ++++++++++++++++++++++++++++ > > > 3 files changed, 50 insertions(+) > > > > > > diff --git a/meta/classes/create-spdx-2.2.bbclass > > > b/meta/classes/create-spdx- > > 2.2.bbclass > > > index 7e8f8b9ff5..6bf0c70bd4 100644 > > > --- a/meta/classes/create-spdx-2.2.bbclass > > > +++ b/meta/classes/create-spdx-2.2.bbclass > > > @@ -137,6 +137,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > > get_spdxid, get_types, *, archiv > > > spdx_files = [] > > > > > > file_counter = 1 > > > + > > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > > + if check_kernel_compiled: > > > + kernel_sources = > > > + oe.spdx_common.get_kernel_compiled_files(d) > > > for subdir, dirs, files in os.walk(topdir): > > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > > if subdir == str(topdir): > > > @@ -147,6 +151,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, > > get_spdxid, get_types, *, archiv > > > filename = str(filepath.relative_to(topdir)) > > > > > > if not filepath.is_symlink() and filepath.is_file(): > > > + # When creating spdx for the kernel, we only include compiled files. > > > + if check_kernel_compiled: > > > + if not oe.spdx_common.is_kernel_compiled(file, > > > + kernel_sources, > > d): > > > + break > > > spdx_file = oe.spdx.SPDXFile() > > > spdx_file.SPDXID = get_spdxid(file_counter) > > > for t in get_types(filepath): > > > diff --git a/meta/lib/oe/spdx30_tasks.py > > > b/meta/lib/oe/spdx30_tasks.py index ba965821f8..14f26773c5 100644 > > > --- a/meta/lib/oe/spdx30_tasks.py > > > +++ b/meta/lib/oe/spdx30_tasks.py > > > @@ -156,6 +156,10 @@ def add_package_files( > > > bb.note(f"Skip {topdir}") > > > return spdx_files > > > > > > + check_kernel_compiled = bb.data.inherits_class("kernel", d) and > > d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" > > > + if check_kernel_compiled: > > > + kernel_sources = > > > + oe.spdx_common.get_kernel_compiled_files(d) > > > + > > > for subdir, dirs, files in os.walk(topdir, onerror=walk_error): > > > dirs[:] = [d for d in dirs if d not in ignore_dirs] > > > if subdir == str(topdir): > > > @@ -167,6 +171,10 @@ def add_package_files( > > > filepath = Path(subdir) / file > > > if filepath.is_symlink() or not filepath.is_file(): > > > continue > > > + # When creating spdx for the kernel, we only include compiled files > > > + if check_kernel_compiled: > > > + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): > > > + break > > > > > > filename = str(filepath.relative_to(topdir)) > > > file_purposes = get_purposes(filepath) diff --git > > > a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py index > > > 4caefc7673..c87e3875c7 100644 > > > --- a/meta/lib/oe/spdx_common.py > > > +++ b/meta/lib/oe/spdx_common.py > > > @@ -242,3 +242,37 @@ def fetch_data_to_uri(fd, name): > > > uri = uri + "@" + fd.revision > > > > > > return uri > > > + > > > +def is_kernel_compiled(filename, kernel_sources, d): > > > + """ > > > + Check if the file, is a kernel compiled file > > > + """ > > > + import os > > > + > > > + _, extension = os.path.splitext(filename) > > > + # Special case, that we need to ignore, since this is not a source file > > > + if filename.rfind(".mod.c") > 0: > > > + return True > > > + # We filter .c files and header files > > > + if extension not in [".c", ".h"]: > > > + return True > > > + # Check that the c file is in the list > > > + if filename in kernel_sources: > > > + return True > > > + return False > > > + > > > +def get_kernel_compiled_files(d): > > > + """ > > > + Get results from the save_compiled files and include also header files > > > + """ > > > + import json > > > + import os > > > + kfiles = [] > > > + with open(d.getVar('KERNEL_SRC_FILES'), 'r') as f: > > > + for item in json.load(f): > > > + kfile = os.path.basename(item['file']) > > > + # Return also the correspondig header file > > > + hfile = ".h".join(kfile.rsplit(".c", 1)) > > > > This feels a little bit wrong to me. I think a better option would be > > to include all headers, since we don't have a good way of knowing > > which are actually used. > > > > > + kfiles.append(kfile) > > > + kfiles.append(hfile) > > > + return kfiles
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass index 7e8f8b9ff5..6bf0c70bd4 100644 --- a/meta/classes/create-spdx-2.2.bbclass +++ b/meta/classes/create-spdx-2.2.bbclass @@ -137,6 +137,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv spdx_files = [] file_counter = 1 + + check_kernel_compiled = bb.data.inherits_class("kernel", d) and d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" + if check_kernel_compiled: + kernel_sources = oe.spdx_common.get_kernel_compiled_files(d) for subdir, dirs, files in os.walk(topdir): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir): @@ -147,6 +151,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv filename = str(filepath.relative_to(topdir)) if not filepath.is_symlink() and filepath.is_file(): + # When creating spdx for the kernel, we only include compiled files. + if check_kernel_compiled: + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): + break spdx_file = oe.spdx.SPDXFile() spdx_file.SPDXID = get_spdxid(file_counter) for t in get_types(filepath): diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py index ba965821f8..14f26773c5 100644 --- a/meta/lib/oe/spdx30_tasks.py +++ b/meta/lib/oe/spdx30_tasks.py @@ -156,6 +156,10 @@ def add_package_files( bb.note(f"Skip {topdir}") return spdx_files + check_kernel_compiled = bb.data.inherits_class("kernel", d) and d.getVar("CVE_CHECK_KERNEL_CONFIG") == "1" + if check_kernel_compiled: + kernel_sources = oe.spdx_common.get_kernel_compiled_files(d) + for subdir, dirs, files in os.walk(topdir, onerror=walk_error): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir): @@ -167,6 +171,10 @@ def add_package_files( filepath = Path(subdir) / file if filepath.is_symlink() or not filepath.is_file(): continue + # When creating spdx for the kernel, we only include compiled files + if check_kernel_compiled: + if not oe.spdx_common.is_kernel_compiled(file, kernel_sources, d): + break filename = str(filepath.relative_to(topdir)) file_purposes = get_purposes(filepath) diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py index 4caefc7673..c87e3875c7 100644 --- a/meta/lib/oe/spdx_common.py +++ b/meta/lib/oe/spdx_common.py @@ -242,3 +242,37 @@ def fetch_data_to_uri(fd, name): uri = uri + "@" + fd.revision return uri + +def is_kernel_compiled(filename, kernel_sources, d): + """ + Check if the file, is a kernel compiled file + """ + import os + + _, extension = os.path.splitext(filename) + # Special case, that we need to ignore, since this is not a source file + if filename.rfind(".mod.c") > 0: + return True + # We filter .c files and header files + if extension not in [".c", ".h"]: + return True + # Check that the c file is in the list + if filename in kernel_sources: + return True + return False + +def get_kernel_compiled_files(d): + """ + Get results from the save_compiled files and include also header files + """ + import json + import os + kfiles = [] + with open(d.getVar('KERNEL_SRC_FILES'), 'r') as f: + for item in json.load(f): + kfile = os.path.basename(item['file']) + # Return also the correspondig header file + hfile = ".h".join(kfile.rsplit(".c", 1)) + kfiles.append(kfile) + kfiles.append(hfile) + return kfiles