| Message ID | 20251103143157.315178-5-niko.mauno@vaisala.com |
|---|---|
| State | New |
| Headers | show |
| Series | [1/5] cve-update-nvd2-native: pycodestyle fixes | expand |
Thanks for this contribution, it was on my TODO list for too much time. I have also one small comment inline below. Peter > -----Original Message----- > From: Niko Mauno <niko.mauno@vaisala.com> > Sent: Monday, November 3, 2025 15:32 > To: openembedded-core@lists.openembedded.org > Cc: ross.burton@arm.com; rybczynska@gmail.com; Marko, Peter (FT D EU SK > BFS1) <Peter.Marko@siemens.com>; Niko Mauno <niko.mauno@vaisala.com> > Subject: [PATCH 5/5] cve-update: Keep mtime stamp in the database itself > > This should help to avoid problems that will occur if the modification > time of database file itself is altered e.g. by unassociated > process(es) on the file system which hosts the database file. > > Since this change updates the database structure by adding a new table, > bump the 'minor' version number in database file names to enforce full > database fetch. This should also iron out e.g. situation where the > database might have inconspicuously omitted entries due to way in which > the mtime of database file itself was relied upon. > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> > --- > meta/classes/cve-check.bbclass | 2 +- > .../recipes-core/meta/cve-update-db-native.bb | 3 + > meta/recipes-core/meta/cve-update-native.inc | 59 ++++++++++++------- > .../meta/cve-update-nvd2-native.bb | 3 + > 4 files changed, 46 insertions(+), 21 deletions(-) > > diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass > index 259c699af2..7252c4ecdc 100644 > --- a/meta/classes/cve-check.bbclass > +++ b/meta/classes/cve-check.bbclass > @@ -35,7 +35,7 @@ CVE_VERSION ??= "${PV}" > NVD_DB_VERSION ?= "FKIE" > > # Use different file names for each database source, as they synchronize at > different moments, so may be slightly different > -CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if > d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-1.db'}" > +CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-3.db' if > d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-2.db'}" > CVE_CHECK_DB_FETCHER ?= "${@'cve-update-nvd2-native' if > d.getVar('NVD_DB_VERSION') == 'NVD2' else 'cve-update-db-native'}" > CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" > CVE_CHECK_DB_FILE ?= > "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" > diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes- > core/meta/cve-update-db-native.bb > index ca83c80958..c1db67ce55 100644 > --- a/meta/recipes-core/meta/cve-update-db-native.bb > +++ b/meta/recipes-core/meta/cve-update-db-native.bb > @@ -18,6 +18,7 @@ def update_db_file(db_tmp_file, d, *_): > """ > import bb.progress > import bb.utils > + import datetime > from datetime import date > import lzma > import sqlite3 > @@ -31,6 +32,7 @@ def update_db_file(db_tmp_file, d, *_): > initialize_db(conn) > > with bb.progress.ProgressHandler(d) as ph, > open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: > + pre_update_utc_timestamp = > datetime.datetime.now().astimezone(tz=datetime.timezone.utc) > total_years = date.today().year + 1 - YEAR_START > for i, year in enumerate(range(YEAR_START, date.today().year + 1)): > bb.note("Updating %d" % year) > @@ -82,6 +84,7 @@ def update_db_file(db_tmp_file, d, *_): > bb.debug(2, "Already up to date (last modified %s)" % last_modified) > # Update success, set the date to cve_check file. > if year == date.today().year: > + conn.execute("insert into MTIME values (?)", > [pre_update_utc_timestamp.isoformat()]).close() > cve_f.write('CVE database update : %s\n\n' % date.today()) > > conn.commit() > diff --git a/meta/recipes-core/meta/cve-update-native.inc b/meta/recipes- > core/meta/cve-update-native.inc > index 298c89b498..2934f7ad07 100644 > --- a/meta/recipes-core/meta/cve-update-native.inc > +++ b/meta/recipes-core/meta/cve-update-native.inc > @@ -33,6 +33,7 @@ python do_fetch() { > import bb.utils > import bb.progress > import shutil > + import time > > bb.utils.export_proxies(d) > > @@ -46,26 +47,24 @@ python do_fetch() { > > # The NVD database changes once a day, so no need to update more > frequently > # Allow the user to force-update > - try: > - import time > - update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL")) > - if update_interval < 0: > - bb.note("CVE database update skipped") > - if not os.path.exists(db_file): > - bb.error("CVE database %s not present, database fetch/update > skipped" % db_file) > - return > - curr_time = time.time() > - database_time = os.path.getmtime(db_file) > - bb.note("Current time: %s; DB time: %s" % (time.ctime(curr_time), > time.ctime(database_time))) > - if curr_time < database_time: > - bb.warn("Database time is in the future, force DB update") > - database_time = 0 > - elif curr_time - database_time < update_interval: > - bb.note("CVE database recently updated, skipping") > - return > - > - except OSError: > - pass > + update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL")) > + if update_interval < 0: > + bb.note("CVE database update skipped") > + if not os.path.exists(db_file): > + bb.error("CVE database %s not present, database fetch/update skipped" > % db_file) > + return > + > + if os.path.exists(db_file): > + database_time = get_mtime_timestamp_from(db_file) > + if database_time > 0: How can we hit the database_time being <= 0? Did you plan some try/catch in function get_mtime_timestamp_from? > + curr_time = time.time() > + bb.note("Current time: %s; DB time: %s" % (time.ctime(curr_time), > time.ctime(database_time))) > + if curr_time < database_time: > + bb.warn("Database time is in the future, force DB update") > + database_time = 0 > + elif curr_time - database_time < update_interval: > + bb.note("CVE database recently updated, skipping") > + return > > if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): > bb.error("BB_NO_NETWORK attempted to disable fetch, this recipe uses > CVE_DB_UPDATE_INTERVAL to control download, set to '-1' to disable fetch or > update") > @@ -107,10 +106,30 @@ def cleanup_db_download(db_tmp_file): > os.remove(db_tmp_file) > > > +def get_mtime_timestamp_from(db_file): > + """ > + Resolve the time when the CVE database was previously updated > + """ > + import datetime > + import sqlite3 > + > + conn = sqlite3.connect(db_file) > + curs = conn.cursor() > + res = curs.execute("select TIMESTAMP from MTIME order by TIMESTAMP > desc limit 1;") > + latest = res.fetchone()[0] > + latest = datetime.datetime.strptime(latest, '%Y-%m- > %dT%H:%M:%S.%f+00:00') > + latest = latest.astimezone(tz=datetime.timezone.utc) > + curs.close() > + conn.close() > + return latest.timestamp() > + > + > def initialize_db(conn): > with conn: > c = conn.cursor() > > + c.execute("CREATE TABLE IF NOT EXISTS MTIME (TIMESTAMP INT)") > + > c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER > UNIQUE, DATE TEXT)") > > c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, > SUMMARY TEXT, \ > diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb b/meta/recipes- > core/meta/cve-update-nvd2-native.bb > index 01d3e8e754..77d7408b16 100644 > --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb > +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb > @@ -72,6 +72,8 @@ def update_db_file(db_tmp_file, d, database_time): > import sqlite3 > import json > > + pre_update_utc_timestamp = > datetime.datetime.now().astimezone(tz=datetime.timezone.utc) > + > # Connect to database > conn = sqlite3.connect(db_tmp_file) > initialize_db(conn) > @@ -141,6 +143,7 @@ def update_db_file(db_tmp_file, d, database_time): > > # Update success, set the date to cve_check file. > cve_f.write('CVE database update : %s\n\n' % datetime.date.today()) > + conn.execute("insert into MTIME values (?)", > [pre_update_utc_timestamp.isoformat()]).close() > > conn.commit() > conn.close() > -- > 2.47.3
On 11/3/25 17:27, Marko, Peter wrote: >> -----Original Message----- >> ... >> + if os.path.exists(db_file): >> + database_time = get_mtime_timestamp_from(db_file) >> + if database_time > 0: > > How can we hit the database_time being <= 0? > Did you plan some try/catch in function get_mtime_timestamp_from? Indeed you are correct, I had a try/except in get_mtime_timestamp_from (with except path returning zero) which I then removed as redundant. I've now submitted v2 which includes the conclusion of said try/except removal by removing also the database_time check here. Thanks a lot for noticing! -Niko
On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via lists.openembedded.org wrote: > This should help to avoid problems that will occur if the modification > time of database file itself is altered e.g. by unassociated > process(es) on the file system which hosts the database file. > > Since this change updates the database structure by adding a new table, > bump the 'minor' version number in database file names to enforce full > database fetch. This should also iron out e.g. situation where the > database might have inconspicuously omitted entries due to way in which > the mtime of database file itself was relied upon. > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> > --- > meta/classes/cve-check.bbclass | 2 +- > .../recipes-core/meta/cve-update-db-native.bb | 3 + > meta/recipes-core/meta/cve-update-native.inc | 59 ++++++++++++------- > .../meta/cve-update-nvd2-native.bb | 3 + > 4 files changed, 46 insertions(+), 21 deletions(-) I am a bit worried about this since it takes what is a simple mtime comparison and means that to check the database you now have to lock and open it which is comparatively expensive. What kind of "unassociated processes" are updating the mtime? Cheers, Richard
On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie < richard.purdie@linuxfoundation.org> wrote: > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via lists.openembedded.org > wrote: > > This should help to avoid problems that will occur if the modification > > time of database file itself is altered e.g. by unassociated > > process(es) on the file system which hosts the database file. > > > > Since this change updates the database structure by adding a new table, > > bump the 'minor' version number in database file names to enforce full > > database fetch. This should also iron out e.g. situation where the > > database might have inconspicuously omitted entries due to way in which > > the mtime of database file itself was relied upon. > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> > > --- > > meta/classes/cve-check.bbclass | 2 +- > > .../recipes-core/meta/cve-update-db-native.bb | 3 + > > meta/recipes-core/meta/cve-update-native.inc | 59 ++++++++++++------- > > .../meta/cve-update-nvd2-native.bb | 3 + > > 4 files changed, 46 insertions(+), 21 deletions(-) > > I am a bit worried about this since it takes what is a simple mtime > comparison and means that to check the database you now have to lock > and open it which is comparatively expensive. > > What kind of "unassociated processes" are updating the mtime? > > I second Richard here. If we would really want to keep the date somewhere else than the file timestamp, we could create a simple file that stores it, what is way simpler and doesn't require to open the database just to see if you want to update it. I second Richard's question: in what kind of a situation have you seen an issue? Kind regards, Marta
> -----Original Message----- > From: Marta Rybczynska <rybczynska@gmail.com> > Sent: Monday, November 10, 2025 16:00 > To: Richard Purdie <richard.purdie@linuxfoundation.org> > Cc: niko.mauno@vaisala.com; openembedded-core@lists.openembedded.org; > ross.burton@arm.com; Marko, Peter (FT D EU SK BFS1) > <Peter.Marko@siemens.com> > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > database itself > > > > On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie > <richard.purdie@linuxfoundation.org <mailto:richard.purdie@linuxfoundation.org> > > wrote: > > > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via > lists.openembedded.org <http://lists.openembedded.org> wrote: > > This should help to avoid problems that will occur if the modification > > time of database file itself is altered e.g. by unassociated > > process(es) on the file system which hosts the database file. > > > > Since this change updates the database structure by adding a new table, > > bump the 'minor' version number in database file names to enforce full > > database fetch. This should also iron out e.g. situation where the > > database might have inconspicuously omitted entries due to way in > which > > the mtime of database file itself was relied upon. > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com > <mailto:niko.mauno@vaisala.com> > > > --- > > meta/classes/cve-check.bbclass | 2 +- > > .../recipes-core/meta/cve-update-db-native.bb <http://cve-update-db- > native.bb> | 3 + > > meta/recipes-core/meta/cve-update-native.inc | 59 ++++++++++++------- > > .../meta/cve-update-nvd2-native.bb <http://cve-update-nvd2-native.bb> > | 3 + > > 4 files changed, 46 insertions(+), 21 deletions(-) > > I am a bit worried about this since it takes what is a simple mtime > comparison and means that to check the database you now have to lock > and open it which is comparatively expensive. > > What kind of "unassociated processes" are updating the mtime? > > > > > I second Richard here. If we would really want to keep the date somewhere else > than the file > timestamp, we could create a simple file that stores it, what is way simpler and > doesn't require > to open the database just to see if you want to update it. > > I second Richard's question: in what kind of a situation have you seen an issue? The issue is on Yocto AB. It has not updated statistics for more than 5 months now. Looking at the logs, something is updating mtime very frequently... I don't know the AB infrastructure, but it looks like there is some filesystem crawler, maybe backup of sync process. Or the nfs mount is plain broken there? Example is here: https://valkyrie.yocto.io/pub/non-release/20251109-43/testresults/metrics/db-update/ NOTE: Current time: Sun Nov 9 07:17:52 2025; DB time: Sun Nov 9 07:14:51 2025 NOTE: Current time: Sun Nov 9 07:19:22 2025; DB time: Sun Nov 9 07:18:14 2025 2 jobs are 90 seconds from each other, but something managed to change the timestamp in between and only 3.5 minutes after last change! Peter > > Kind regards, > Marta
On Mon, Nov 10, 2025 at 4:18 PM Marko, Peter <Peter.Marko@siemens.com> wrote: > > > > -----Original Message----- > > From: Marta Rybczynska <rybczynska@gmail.com> > > Sent: Monday, November 10, 2025 16:00 > > To: Richard Purdie <richard.purdie@linuxfoundation.org> > > Cc: niko.mauno@vaisala.com; openembedded-core@lists.openembedded.org; > > ross.burton@arm.com; Marko, Peter (FT D EU SK BFS1) > > <Peter.Marko@siemens.com> > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > > database itself > > > > > > > > On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie > > <richard.purdie@linuxfoundation.org <mailto: > richard.purdie@linuxfoundation.org> > > > wrote: > > > > > > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via > > lists.openembedded.org <http://lists.openembedded.org> wrote: > > > This should help to avoid problems that will occur if the > modification > > > time of database file itself is altered e.g. by unassociated > > > process(es) on the file system which hosts the database file. > > > > > > Since this change updates the database structure by adding a new > table, > > > bump the 'minor' version number in database file names to > enforce full > > > database fetch. This should also iron out e.g. situation where > the > > > database might have inconspicuously omitted entries due to way in > > which > > > the mtime of database file itself was relied upon. > > > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com > > <mailto:niko.mauno@vaisala.com> > > > > --- > > > meta/classes/cve-check.bbclass | 2 +- > > > .../recipes-core/meta/cve-update-db-native.bb < > http://cve-update-db- > > native.bb> | 3 + > > > meta/recipes-core/meta/cve-update-native.inc | 59 > ++++++++++++------- > > > .../meta/cve-update-nvd2-native.bb < > http://cve-update-nvd2-native.bb> > > | 3 + > > > 4 files changed, 46 insertions(+), 21 deletions(-) > > > > I am a bit worried about this since it takes what is a simple mtime > > comparison and means that to check the database you now have to > lock > > and open it which is comparatively expensive. > > > > What kind of "unassociated processes" are updating the mtime? > > > > > > > > > > I second Richard here. If we would really want to keep the date > somewhere else > > than the file > > timestamp, we could create a simple file that stores it, what is way > simpler and > > doesn't require > > to open the database just to see if you want to update it. > > > > I second Richard's question: in what kind of a situation have you seen > an issue? > > The issue is on Yocto AB. > It has not updated statistics for more than 5 months now. > > Looking at the logs, something is updating mtime very frequently... > I don't know the AB infrastructure, but it looks like there is some > filesystem crawler, maybe backup of sync process. > Or the nfs mount is plain broken there? > > Example is here: > > https://valkyrie.yocto.io/pub/non-release/20251109-43/testresults/metrics/db-update/ > NOTE: Current time: Sun Nov 9 07:17:52 2025; DB time: Sun Nov 9 07:14:51 > 2025 > NOTE: Current time: Sun Nov 9 07:19:22 2025; DB time: Sun Nov 9 07:18:14 > 2025 > 2 jobs are 90 seconds from each other, but something managed to change the > timestamp in between and only 3.5 minutes after last change! > > The autobuilder was expected to force the complete download every day. Something has changed? The code does open the database after checking the timestamp, so frequent jobs couldn't cause it. I think that machine requires some monitoring to see what is actually changing those timestamps, because we do not know what it is also changing. I haven't seen such an effect on my systems. Kind regards, Marta
> -----Original Message----- > From: Marta Rybczynska <rybczynska@gmail.com> > Sent: Monday, November 10, 2025 16:29 > To: Marko, Peter (FT D EU SK BFS1) <Peter.Marko@siemens.com> > Cc: Richard Purdie <richard.purdie@linuxfoundation.org>; > niko.mauno@vaisala.com; openembedded-core@lists.openembedded.org; > ross.burton@arm.com > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > database itself > > > > On Mon, Nov 10, 2025 at 4:18 PM Marko, Peter <Peter.Marko@siemens.com > <mailto:Peter.Marko@siemens.com> > wrote: > > > > > > -----Original Message----- > > From: Marta Rybczynska <rybczynska@gmail.com > <mailto:rybczynska@gmail.com> > > > Sent: Monday, November 10, 2025 16:00 > > To: Richard Purdie <richard.purdie@linuxfoundation.org > <mailto:richard.purdie@linuxfoundation.org> > > > Cc: niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> ; > openembedded-core@lists.openembedded.org <mailto:openembedded- > core@lists.openembedded.org> ; > > ross.burton@arm.com <mailto:ross.burton@arm.com> ; Marko, Peter > (FT D EU SK BFS1) > > <Peter.Marko@siemens.com <mailto:Peter.Marko@siemens.com> > > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in > the > > database itself > > > > > > > > On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie > > <richard.purdie@linuxfoundation.org > <mailto:richard.purdie@linuxfoundation.org> > <mailto:richard.purdie@linuxfoundation.org > <mailto:richard.purdie@linuxfoundation.org> > > > > wrote: > > > > > > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via > > lists.openembedded.org <http://lists.openembedded.org> > <http://lists.openembedded.org> wrote: > > > This should help to avoid problems that will occur if the > modification > > > time of database file itself is altered e.g. by unassociated > > > process(es) on the file system which hosts the database file. > > > > > > Since this change updates the database structure by adding a new > table, > > > bump the 'minor' version number in database file names to enforce > full > > > database fetch. This should also iron out e.g. situation where the > > > database might have inconspicuously omitted entries due to way > in > > which > > > the mtime of database file itself was relied upon. > > > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com > <mailto:niko.mauno@vaisala.com> > > <mailto:niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> > > > > > > --- > > > meta/classes/cve-check.bbclass | 2 +- > > > .../recipes-core/meta/cve-update-db-native.bb <http://cve-update- > db-native.bb> <http://cve-update-db- > > native.bb <http://native.bb> > | 3 + > > > meta/recipes-core/meta/cve-update-native.inc | 59 > ++++++++++++------- > > > .../meta/cve-update-nvd2-native.bb <http://cve-update-nvd2- > native.bb> <http://cve-update-nvd2-native.bb> > > | 3 + > > > 4 files changed, 46 insertions(+), 21 deletions(-) > > > > I am a bit worried about this since it takes what is a simple mtime > > comparison and means that to check the database you now have to > lock > > and open it which is comparatively expensive. > > > > What kind of "unassociated processes" are updating the mtime? > > > > > > > > > > I second Richard here. If we would really want to keep the date > somewhere else > > than the file > > timestamp, we could create a simple file that stores it, what is way > simpler and > > doesn't require > > to open the database just to see if you want to update it. > > > > I second Richard's question: in what kind of a situation have you seen an > issue? > > The issue is on Yocto AB. > It has not updated statistics for more than 5 months now. > > Looking at the logs, something is updating mtime very frequently... > I don't know the AB infrastructure, but it looks like there is some filesystem > crawler, maybe backup of sync process. > Or the nfs mount is plain broken there? > > Example is here: > https://valkyrie.yocto.io/pub/non-release/20251109- > 43/testresults/metrics/db-update/ > NOTE: Current time: Sun Nov 9 07:17:52 2025; DB time: Sun Nov 9 > 07:14:51 2025 > NOTE: Current time: Sun Nov 9 07:19:22 2025; DB time: Sun Nov 9 > 07:18:14 2025 > 2 jobs are 90 seconds from each other, but something managed to change > the timestamp in between and only 3.5 minutes after last change! > > > > > The autobuilder was expected to force the complete download every day. > Something > has changed? The code does open the database after checking the timestamp, so > frequent jobs couldn't cause it. I think that machine requires some monitoring to > see > what is actually changing those timestamps, because we do not know what it is > also > changing. 1 day doesn't beat the 3.5 minutes crawler... This is probably AB issue only. And maybe other example: https://valkyrie.yocto.io/pub/non-release/20251110-45/testresults/metrics/db-update/ NOTE: Current time: Mon Nov 10 07:17:47 2025; DB time: Mon Nov 10 07:16:33 2025 NOTE: Current time: Mon Nov 10 07:19:17 2025; DB time: Mon Nov 10 07:16:33 2025 This shows that the first job does not modify it (at least not in the first 90 seconds). I guess it fit the 3.5 minutes window of opportunity. And my local build does not show any timestamp manipulation either. Peter > > I haven't seen such an effect on my systems. > > Kind regards, > Marta
On Mon, Nov 10, 2025 at 7:31 AM Peter Marko via lists.openembedded.org <peter.marko=siemens.com@lists.openembedded.org> wrote: > > > > > -----Original Message----- > > From: Marta Rybczynska <rybczynska@gmail.com> > > Sent: Monday, November 10, 2025 16:29 > > To: Marko, Peter (FT D EU SK BFS1) <Peter.Marko@siemens.com> > > Cc: Richard Purdie <richard.purdie@linuxfoundation.org>; > > niko.mauno@vaisala.com; openembedded-core@lists.openembedded.org; > > ross.burton@arm.com > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > > database itself > > > > > > > > On Mon, Nov 10, 2025 at 4:18 PM Marko, Peter <Peter.Marko@siemens.com > > <mailto:Peter.Marko@siemens.com> > wrote: > > > > > > > > > > > -----Original Message----- > > > From: Marta Rybczynska <rybczynska@gmail.com > > <mailto:rybczynska@gmail.com> > > > > Sent: Monday, November 10, 2025 16:00 > > > To: Richard Purdie <richard.purdie@linuxfoundation.org > > <mailto:richard.purdie@linuxfoundation.org> > > > > Cc: niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> ; > > openembedded-core@lists.openembedded.org <mailto:openembedded- > > core@lists.openembedded.org> ; > > > ross.burton@arm.com <mailto:ross.burton@arm.com> ; Marko, Peter > > (FT D EU SK BFS1) > > > <Peter.Marko@siemens.com <mailto:Peter.Marko@siemens.com> > > > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in > > the > > > database itself > > > > > > > > > > > > On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie > > > <richard.purdie@linuxfoundation.org > > <mailto:richard.purdie@linuxfoundation.org> > > <mailto:richard.purdie@linuxfoundation.org > > <mailto:richard.purdie@linuxfoundation.org> > > > > > wrote: > > > > > > > > > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via > > > lists.openembedded.org <http://lists.openembedded.org> > > <http://lists.openembedded.org> wrote: > > > > This should help to avoid problems that will occur if the > > modification > > > > time of database file itself is altered e.g. by unassociated > > > > process(es) on the file system which hosts the database file. > > > > > > > > Since this change updates the database structure by adding a new > > table, > > > > bump the 'minor' version number in database file names to enforce > > full > > > > database fetch. This should also iron out e.g. situation where the > > > > database might have inconspicuously omitted entries due to way > > in > > > which > > > > the mtime of database file itself was relied upon. > > > > > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com > > <mailto:niko.mauno@vaisala.com> > > > <mailto:niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> > > > > > > > > --- > > > > meta/classes/cve-check.bbclass | 2 +- > > > > .../recipes-core/meta/cve-update-db-native.bb <http://cve-update- > > db-native.bb> <http://cve-update-db- > > > native.bb <http://native.bb> > | 3 + > > > > meta/recipes-core/meta/cve-update-native.inc | 59 > > ++++++++++++------- > > > > .../meta/cve-update-nvd2-native.bb <http://cve-update-nvd2- > > native.bb> <http://cve-update-nvd2-native.bb> > > > | 3 + > > > > 4 files changed, 46 insertions(+), 21 deletions(-) > > > > > > I am a bit worried about this since it takes what is a simple mtime > > > comparison and means that to check the database you now have to > > lock > > > and open it which is comparatively expensive. > > > > > > What kind of "unassociated processes" are updating the mtime? > > > > > > > > > > > > > > > I second Richard here. If we would really want to keep the date > > somewhere else > > > than the file > > > timestamp, we could create a simple file that stores it, what is way > > simpler and > > > doesn't require > > > to open the database just to see if you want to update it. > > > > > > I second Richard's question: in what kind of a situation have you seen an > > issue? > > > > The issue is on Yocto AB. > > It has not updated statistics for more than 5 months now. > > > > Looking at the logs, something is updating mtime very frequently... > > I don't know the AB infrastructure, but it looks like there is some filesystem > > crawler, maybe backup of sync process. On 2024-12-26 we set workers in the autobulder.yoctoproject.org/valkyrie cluster to run "touch /srv/autobuilder/valkyrie.yocto.io/current_sources/CVE_CHECK/*.db /srv/autobuilder/valkyrie.yocto.io/current_sources/CVE_CHECK2/*.db" every 3 minute to ensure no stale NFS cache existed for these files. That job was disabled for all builders on 2025-09-02. The performance testing workers were still running the job though. I've disabled it on those now. The issue of stale cached NFS data may return now. Something to keep an eye on. If we want to keep the timestamp in a separate file I suggest creating a new file named timestamp_$(date +%s) for each timestamp and deleting previous timestamp_ files. That should avoid most caching issues and keep overhead minimal. > > Or the nfs mount is plain broken there? > > > > Example is here: > > https://valkyrie.yocto.io/pub/non-release/20251109- > > 43/testresults/metrics/db-update/ > > NOTE: Current time: Sun Nov 9 07:17:52 2025; DB time: Sun Nov 9 > > 07:14:51 2025 > > NOTE: Current time: Sun Nov 9 07:19:22 2025; DB time: Sun Nov 9 > > 07:18:14 2025 > > 2 jobs are 90 seconds from each other, but something managed to change > > the timestamp in between and only 3.5 minutes after last change! > > > > > > > > > > The autobuilder was expected to force the complete download every day. > > Something > > has changed? The code does open the database after checking the timestamp, so > > frequent jobs couldn't cause it. I think that machine requires some monitoring to > > see > > what is actually changing those timestamps, because we do not know what it is > > also > > changing. > > 1 day doesn't beat the 3.5 minutes crawler... > This is probably AB issue only. > > And maybe other example: > https://valkyrie.yocto.io/pub/non-release/20251110-45/testresults/metrics/db-update/ > NOTE: Current time: Mon Nov 10 07:17:47 2025; DB time: Mon Nov 10 07:16:33 2025 > NOTE: Current time: Mon Nov 10 07:19:17 2025; DB time: Mon Nov 10 07:16:33 2025 > > This shows that the first job does not modify it (at least not in the first 90 seconds). > I guess it fit the 3.5 minutes window of opportunity. > And my local build does not show any timestamp manipulation either. > > Peter > > > > > I haven't seen such an effect on my systems. > > > > Kind regards, > > Marta > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#226127): https://lists.openembedded.org/g/openembedded-core/message/226127 > Mute This Topic: https://lists.openembedded.org/mt/116098343/1003190 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [mhalstead@linuxfoundation.org] > -=-=-=-=-=-=-=-=-=-=-=- >
> -----Original Message----- > From: Michael Halstead <mhalstead@linuxfoundation.org> > Sent: Tuesday, November 11, 2025 6:31 > To: Marko, Peter (FT D EU SK BFS1) <Peter.Marko@siemens.com> > Cc: Marta Rybczynska <rybczynska@gmail.com>; Richard Purdie > <richard.purdie@linuxfoundation.org>; niko.mauno@vaisala.com; openembedded- > core@lists.openembedded.org; ross.burton@arm.com > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > database itself > > On Mon, Nov 10, 2025 at 7:31 AM Peter Marko via lists.openembedded.org > <peter.marko=siemens.com@lists.openembedded.org> wrote: > > > > > > > > > -----Original Message----- > > > From: Marta Rybczynska <rybczynska@gmail.com> > > > Sent: Monday, November 10, 2025 16:29 > > > To: Marko, Peter (FT D EU SK BFS1) <Peter.Marko@siemens.com> > > > Cc: Richard Purdie <richard.purdie@linuxfoundation.org>; > > > niko.mauno@vaisala.com; openembedded-core@lists.openembedded.org; > > > ross.burton@arm.com > > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > > > database itself > > > > > > > > > > > > On Mon, Nov 10, 2025 at 4:18 PM Marko, Peter <Peter.Marko@siemens.com > > > <mailto:Peter.Marko@siemens.com> > wrote: > > > > > > > > > > > > > > > > -----Original Message----- > > > > From: Marta Rybczynska <rybczynska@gmail.com > > > <mailto:rybczynska@gmail.com> > > > > > Sent: Monday, November 10, 2025 16:00 > > > > To: Richard Purdie <richard.purdie@linuxfoundation.org > > > <mailto:richard.purdie@linuxfoundation.org> > > > > > Cc: niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> ; > > > openembedded-core@lists.openembedded.org <mailto:openembedded- > > > core@lists.openembedded.org> ; > > > > ross.burton@arm.com <mailto:ross.burton@arm.com> ; Marko, Peter > > > (FT D EU SK BFS1) > > > > <Peter.Marko@siemens.com <mailto:Peter.Marko@siemens.com> > > > > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in > > > the > > > > database itself > > > > > > > > > > > > > > > > On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie > > > > <richard.purdie@linuxfoundation.org > > > <mailto:richard.purdie@linuxfoundation.org> > > > <mailto:richard.purdie@linuxfoundation.org > > > <mailto:richard.purdie@linuxfoundation.org> > > > > > > wrote: > > > > > > > > > > > > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via > > > > lists.openembedded.org <http://lists.openembedded.org> > > > <http://lists.openembedded.org> wrote: > > > > > This should help to avoid problems that will occur if the > > > modification > > > > > time of database file itself is altered e.g. by unassociated > > > > > process(es) on the file system which hosts the database file. > > > > > > > > > > Since this change updates the database structure by adding a new > > > table, > > > > > bump the 'minor' version number in database file names to enforce > > > full > > > > > database fetch. This should also iron out e.g. situation where the > > > > > database might have inconspicuously omitted entries due to way > > > in > > > > which > > > > > the mtime of database file itself was relied upon. > > > > > > > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com > > > <mailto:niko.mauno@vaisala.com> > > > > <mailto:niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> > > > > > > > > > > --- > > > > > meta/classes/cve-check.bbclass | 2 +- > > > > > .../recipes-core/meta/cve-update-db-native.bb <http://cve-update- > > > db-native.bb> <http://cve-update-db- > > > > native.bb <http://native.bb> > | 3 + > > > > > meta/recipes-core/meta/cve-update-native.inc | 59 > > > ++++++++++++------- > > > > > .../meta/cve-update-nvd2-native.bb <http://cve-update-nvd2- > > > native.bb> <http://cve-update-nvd2-native.bb> > > > > | 3 + > > > > > 4 files changed, 46 insertions(+), 21 deletions(-) > > > > > > > > I am a bit worried about this since it takes what is a simple mtime > > > > comparison and means that to check the database you now have to > > > lock > > > > and open it which is comparatively expensive. > > > > > > > > What kind of "unassociated processes" are updating the mtime? > > > > > > > > > > > > > > > > > > > > I second Richard here. If we would really want to keep the date > > > somewhere else > > > > than the file > > > > timestamp, we could create a simple file that stores it, what is way > > > simpler and > > > > doesn't require > > > > to open the database just to see if you want to update it. > > > > > > > > I second Richard's question: in what kind of a situation have you seen an > > > issue? > > > > > > The issue is on Yocto AB. > > > It has not updated statistics for more than 5 months now. > > > > > > Looking at the logs, something is updating mtime very frequently... > > > I don't know the AB infrastructure, but it looks like there is some filesystem > > > crawler, maybe backup of sync process. > > On 2024-12-26 we set workers in the > autobulder.yoctoproject.org/valkyrie cluster to run "touch > /srv/autobuilder/valkyrie.yocto.io/current_sources/CVE_CHECK/*.db > /srv/autobuilder/valkyrie.yocto.io/current_sources/CVE_CHECK2/*.db" > every 3 minute to ensure no stale NFS cache existed for these files. This explains a lot. Touching file while we rely on mtime is obviously breaking cve-check code. Note that stopping the touch service is not enough, at least not for LTS branches, because NVD2 code will update new CVEs only since the mtime timestamp. Could you also delete the databases now so it can start fresh without 5 months of missing data? Peter > That job was disabled for all builders on 2025-09-02. The performance > testing workers were still running the job though. I've disabled it on > those now. > > The issue of stale cached NFS data may return now. Something to keep an eye on. > > If we want to keep the timestamp in a separate file I suggest creating > a new file named timestamp_$(date +%s) for each timestamp and deleting > previous timestamp_ files. That should avoid most caching issues and > keep overhead minimal. > > > > Or the nfs mount is plain broken there? > > > > > > Example is here: > > > https://valkyrie.yocto.io/pub/non-release/20251109- > > > 43/testresults/metrics/db-update/ > > > NOTE: Current time: Sun Nov 9 07:17:52 2025; DB time: Sun Nov 9 > > > 07:14:51 2025 > > > NOTE: Current time: Sun Nov 9 07:19:22 2025; DB time: Sun Nov 9 > > > 07:18:14 2025 > > > 2 jobs are 90 seconds from each other, but something managed to change > > > the timestamp in between and only 3.5 minutes after last change! > > > > > > > > > > > > > > > The autobuilder was expected to force the complete download every day. > > > Something > > > has changed? The code does open the database after checking the > timestamp, so > > > frequent jobs couldn't cause it. I think that machine requires some monitoring > to > > > see > > > what is actually changing those timestamps, because we do not know what it > is > > > also > > > changing. > > > > 1 day doesn't beat the 3.5 minutes crawler... > > This is probably AB issue only. > > > > And maybe other example: > > https://valkyrie.yocto.io/pub/non-release/20251110-45/testresults/metrics/db- > update/ > > NOTE: Current time: Mon Nov 10 07:17:47 2025; DB time: Mon Nov 10 07:16:33 > 2025 > > NOTE: Current time: Mon Nov 10 07:19:17 2025; DB time: Mon Nov 10 07:16:33 > 2025 > > > > This shows that the first job does not modify it (at least not in the first 90 > seconds). > > I guess it fit the 3.5 minutes window of opportunity. > > And my local build does not show any timestamp manipulation either. > > > > Peter > > > > > > > > I haven't seen such an effect on my systems. > > > > > > Kind regards, > > > Marta > > > > > > -=-=-=-=-=-=-=-=-=-=-=- > > Links: You receive all messages sent to this group. > > View/Reply Online (#226127): https://lists.openembedded.org/g/openembedded- > core/message/226127 > > Mute This Topic: https://lists.openembedded.org/mt/116098343/1003190 > > Group Owner: openembedded-core+owner@lists.openembedded.org > > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub > [mhalstead@linuxfoundation.org] > > -=-=-=-=-=-=-=-=-=-=-=- > > > > > -- > Michael Halstead > Linux Foundation / Yocto Project > Systems Operations Engineer
On Mon, Nov 10, 2025 at 11:42 PM Peter Marko via lists.openembedded.org <peter.marko=siemens.com@lists.openembedded.org> wrote: > > > > > -----Original Message----- > > From: Michael Halstead <mhalstead@linuxfoundation.org> > > Sent: Tuesday, November 11, 2025 6:31 > > To: Marko, Peter (FT D EU SK BFS1) <Peter.Marko@siemens.com> > > Cc: Marta Rybczynska <rybczynska@gmail.com>; Richard Purdie > > <richard.purdie@linuxfoundation.org>; niko.mauno@vaisala.com; openembedded- > > core@lists.openembedded.org; ross.burton@arm.com > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > > database itself > > > > On Mon, Nov 10, 2025 at 7:31 AM Peter Marko via lists.openembedded.org > > <peter.marko=siemens.com@lists.openembedded.org> wrote: > > > > > > > > > > > > > -----Original Message----- > > > > From: Marta Rybczynska <rybczynska@gmail.com> > > > > Sent: Monday, November 10, 2025 16:29 > > > > To: Marko, Peter (FT D EU SK BFS1) <Peter.Marko@siemens.com> > > > > Cc: Richard Purdie <richard.purdie@linuxfoundation.org>; > > > > niko.mauno@vaisala.com; openembedded-core@lists.openembedded.org; > > > > ross.burton@arm.com > > > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in the > > > > database itself > > > > > > > > > > > > > > > > On Mon, Nov 10, 2025 at 4:18 PM Marko, Peter <Peter.Marko@siemens.com > > > > <mailto:Peter.Marko@siemens.com> > wrote: > > > > > > > > > > > > > > > > > > > > > -----Original Message----- > > > > > From: Marta Rybczynska <rybczynska@gmail.com > > > > <mailto:rybczynska@gmail.com> > > > > > > Sent: Monday, November 10, 2025 16:00 > > > > > To: Richard Purdie <richard.purdie@linuxfoundation.org > > > > <mailto:richard.purdie@linuxfoundation.org> > > > > > > Cc: niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> ; > > > > openembedded-core@lists.openembedded.org <mailto:openembedded- > > > > core@lists.openembedded.org> ; > > > > > ross.burton@arm.com <mailto:ross.burton@arm.com> ; Marko, Peter > > > > (FT D EU SK BFS1) > > > > > <Peter.Marko@siemens.com <mailto:Peter.Marko@siemens.com> > > > > > > Subject: Re: [OE-core] [PATCH 5/5] cve-update: Keep mtime stamp in > > > > the > > > > > database itself > > > > > > > > > > > > > > > > > > > > On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie > > > > > <richard.purdie@linuxfoundation.org > > > > <mailto:richard.purdie@linuxfoundation.org> > > > > <mailto:richard.purdie@linuxfoundation.org > > > > <mailto:richard.purdie@linuxfoundation.org> > > > > > > > wrote: > > > > > > > > > > > > > > > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via > > > > > lists.openembedded.org <http://lists.openembedded.org> > > > > <http://lists.openembedded.org> wrote: > > > > > > This should help to avoid problems that will occur if the > > > > modification > > > > > > time of database file itself is altered e.g. by unassociated > > > > > > process(es) on the file system which hosts the database file. > > > > > > > > > > > > Since this change updates the database structure by adding a new > > > > table, > > > > > > bump the 'minor' version number in database file names to enforce > > > > full > > > > > > database fetch. This should also iron out e.g. situation where the > > > > > > database might have inconspicuously omitted entries due to way > > > > in > > > > > which > > > > > > the mtime of database file itself was relied upon. > > > > > > > > > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com > > > > <mailto:niko.mauno@vaisala.com> > > > > > <mailto:niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com> > > > > > > > > > > > > --- > > > > > > meta/classes/cve-check.bbclass | 2 +- > > > > > > .../recipes-core/meta/cve-update-db-native.bb <http://cve-update- > > > > db-native.bb> <http://cve-update-db- > > > > > native.bb <http://native.bb> > | 3 + > > > > > > meta/recipes-core/meta/cve-update-native.inc | 59 > > > > ++++++++++++------- > > > > > > .../meta/cve-update-nvd2-native.bb <http://cve-update-nvd2- > > > > native.bb> <http://cve-update-nvd2-native.bb> > > > > > | 3 + > > > > > > 4 files changed, 46 insertions(+), 21 deletions(-) > > > > > > > > > > I am a bit worried about this since it takes what is a simple mtime > > > > > comparison and means that to check the database you now have to > > > > lock > > > > > and open it which is comparatively expensive. > > > > > > > > > > What kind of "unassociated processes" are updating the mtime? > > > > > > > > > > > > > > > > > > > > > > > > > I second Richard here. If we would really want to keep the date > > > > somewhere else > > > > > than the file > > > > > timestamp, we could create a simple file that stores it, what is way > > > > simpler and > > > > > doesn't require > > > > > to open the database just to see if you want to update it. > > > > > > > > > > I second Richard's question: in what kind of a situation have you seen an > > > > issue? > > > > > > > > The issue is on Yocto AB. > > > > It has not updated statistics for more than 5 months now. > > > > > > > > Looking at the logs, something is updating mtime very frequently... > > > > I don't know the AB infrastructure, but it looks like there is some filesystem > > > > crawler, maybe backup of sync process. > > > > On 2024-12-26 we set workers in the > > autobulder.yoctoproject.org/valkyrie cluster to run "touch > > /srv/autobuilder/valkyrie.yocto.io/current_sources/CVE_CHECK/*.db > > /srv/autobuilder/valkyrie.yocto.io/current_sources/CVE_CHECK2/*.db" > > every 3 minute to ensure no stale NFS cache existed for these files. > > This explains a lot. > Touching file while we rely on mtime is obviously breaking cve-check code. > > Note that stopping the touch service is not enough, at least not for LTS branches, > because NVD2 code will update new CVEs only since the mtime timestamp. > Could you also delete the databases now so it can start fresh without 5 months of missing data? I've removed those files. > > Peter > > > That job was disabled for all builders on 2025-09-02. The performance > > testing workers were still running the job though. I've disabled it on > > those now. > > > > The issue of stale cached NFS data may return now. Something to keep an eye on. > > > > If we want to keep the timestamp in a separate file I suggest creating > > a new file named timestamp_$(date +%s) for each timestamp and deleting > > previous timestamp_ files. That should avoid most caching issues and > > keep overhead minimal. > > > > > > Or the nfs mount is plain broken there? > > > > > > > > Example is here: > > > > https://valkyrie.yocto.io/pub/non-release/20251109- > > > > 43/testresults/metrics/db-update/ > > > > NOTE: Current time: Sun Nov 9 07:17:52 2025; DB time: Sun Nov 9 > > > > 07:14:51 2025 > > > > NOTE: Current time: Sun Nov 9 07:19:22 2025; DB time: Sun Nov 9 > > > > 07:18:14 2025 > > > > 2 jobs are 90 seconds from each other, but something managed to change > > > > the timestamp in between and only 3.5 minutes after last change! > > > > > > > > > > > > > > > > > > > > The autobuilder was expected to force the complete download every day. > > > > Something > > > > has changed? The code does open the database after checking the > > timestamp, so > > > > frequent jobs couldn't cause it. I think that machine requires some monitoring > > to > > > > see > > > > what is actually changing those timestamps, because we do not know what it > > is > > > > also > > > > changing. > > > > > > 1 day doesn't beat the 3.5 minutes crawler... > > > This is probably AB issue only. > > > > > > And maybe other example: > > > https://valkyrie.yocto.io/pub/non-release/20251110-45/testresults/metrics/db- > > update/ > > > NOTE: Current time: Mon Nov 10 07:17:47 2025; DB time: Mon Nov 10 07:16:33 > > 2025 > > > NOTE: Current time: Mon Nov 10 07:19:17 2025; DB time: Mon Nov 10 07:16:33 > > 2025 > > > > > > This shows that the first job does not modify it (at least not in the first 90 > > seconds). > > > I guess it fit the 3.5 minutes window of opportunity. > > > And my local build does not show any timestamp manipulation either. > > > > > > Peter > > > > > > > > > > > I haven't seen such an effect on my systems. > > > > > > > > Kind regards, > > > > Marta > > > > > > > > > > > > > > > > > > -- > > Michael Halstead > > Linux Foundation / Yocto Project > > Systems Operations Engineer > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#226148): https://lists.openembedded.org/g/openembedded-core/message/226148 > Mute This Topic: https://lists.openembedded.org/mt/116098343/1003190 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [mhalstead@linuxfoundation.org] > -=-=-=-=-=-=-=-=-=-=-=- >
On 11/10/25 16:59, Marta Rybczynska wrote: > > > On Mon, Nov 10, 2025 at 3:16 PM Richard Purdie <richard.purdie@linuxfoundation.org <mailto:richard.purdie@linuxfoundation.org>> wrote: > > On Mon, 2025-11-03 at 14:31 +0000, Niko Mauno via lists.openembedded.org <http://lists.openembedded.org/> wrote: > > This should help to avoid problems that will occur if the modification > > time of database file itself is altered e.g. by unassociated > > process(es) on the file system which hosts the database file. > > > > Since this change updates the database structure by adding a new table, > > bump the 'minor' version number in database file names to enforce full > > database fetch. This should also iron out e.g. situation where the > > database might have inconspicuously omitted entries due to way in which > > the mtime of database file itself was relied upon. > > > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com <mailto:niko.mauno@vaisala.com>> > > --- > > meta/classes/cve-check.bbclass | 2 +- > > .../recipes-core/meta/cve-update-db-native.bb <http://cve-update-db-native.bb/> | 3 + > > meta/recipes-core/meta/cve-update-native.inc | 59 ++++++++++++------- > > .../meta/cve-update-nvd2-native.bb <http://cve-update-nvd2-native.bb/> | 3 + > > 4 files changed, 46 insertions(+), 21 deletions(-) > > I am a bit worried about this since it takes what is a simple mtime > comparison and means that to check the database you now have to lock > and open it which is comparatively expensive. Hello Richard, To me it seemed that the overhead of the additional database access which is performed once or twice during BitBake execution lifecycle is really neglible, and in relative terms, very cheap compared to producing false negative CVE checks. This change also hopes to close a window of opportunity by sampling the database mtime timestamp value before the fetch operation is performed, in contrast to former when the file mtime is (supposedly) updated only after the fetch has been performed and the database write completed. > What kind of "unassociated processes" are updating the mtime? Seems this question was kindly filled in by Peter and Michael on AB behalf -- for my own behalf, please see the response below in Marta's "seconding" question. > > I second Richard here. If we would really want to keep the date somewhere else than the file > timestamp, we could create a simple file that stores it, what is way simpler and doesn't require > to open the database just to see if you want to update it. Hello Marta, I fail to see what is the real benefit in using a separate file for the purpose, and somehow I doubt the operations involved would be that much simpler, or provide any significant edge performance wise. With this change we just move away from using file mtime and still keep the timestamp along in the one and same database file. > I second Richard's question: in what kind of a situation have you seen an issue? Personally I was bitten by the file mtime dependency about an year ago when there were serious issues with downloading the database updates from the NVD website. I tried to mitigate the problem by using a centralized job for downloading the database (with very lenient settings so it wouldn't time out or so) and then distributed the database file to several build servers. But with this approach the mtime value was always recently updated before the database update attempt, and it took a while for me to realize the mtime check prevented the database update from occurring. I don't recall what was the cause of the file mtime getting updated at that time, the database file was compressed+decompressed as part of the flow and resided behind NFS mount. Later I also noticed that others seemed to have encountered issues which could also be related to use of "external" file mtime provided timestamp - https://lists.openembedded.org/g/openembedded-core/topic/110179038#msg208869 - https://git.yoctoproject.org/poky/commit/?id=9d2dcd798322748cfdead0db320a5242e4b2cf31 These made me wonder if there could be some less flaky alternative to the file mtime reliance, which seemed under some circumstances to lead to inconspicuous issues in form of false negative CVE checks due to omitted CVEs in the database. > Kind regards, > Marta Finally, considering Marta's advise in her other reply in this thread, quoting her comment here: > So for people who want to use the NVD data directly from the source still, I think it makes sense to implement the download > of the JSON 2.0 feeds (should be closer to the nvd-native than nvd2-native, and *faster*) and then decommission both nvd1 > and the current nvd2 (API based). If it helps I can e.g. rework the series such that only the changes in the topmost "keep mtime in database itself" commit would be applied in the two recipes independently, which would still help to avoid the current mtime check from occasionally biting unsuspecting users. -Niko
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass index 259c699af2..7252c4ecdc 100644 --- a/meta/classes/cve-check.bbclass +++ b/meta/classes/cve-check.bbclass @@ -35,7 +35,7 @@ CVE_VERSION ??= "${PV}" NVD_DB_VERSION ?= "FKIE" # Use different file names for each database source, as they synchronize at different moments, so may be slightly different -CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-1.db'}" +CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-3.db' if d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-2.db'}" CVE_CHECK_DB_FETCHER ?= "${@'cve-update-nvd2-native' if d.getVar('NVD_DB_VERSION') == 'NVD2' else 'cve-update-db-native'}" CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb index ca83c80958..c1db67ce55 100644 --- a/meta/recipes-core/meta/cve-update-db-native.bb +++ b/meta/recipes-core/meta/cve-update-db-native.bb @@ -18,6 +18,7 @@ def update_db_file(db_tmp_file, d, *_): """ import bb.progress import bb.utils + import datetime from datetime import date import lzma import sqlite3 @@ -31,6 +32,7 @@ def update_db_file(db_tmp_file, d, *_): initialize_db(conn) with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: + pre_update_utc_timestamp = datetime.datetime.now().astimezone(tz=datetime.timezone.utc) total_years = date.today().year + 1 - YEAR_START for i, year in enumerate(range(YEAR_START, date.today().year + 1)): bb.note("Updating %d" % year) @@ -82,6 +84,7 @@ def update_db_file(db_tmp_file, d, *_): bb.debug(2, "Already up to date (last modified %s)" % last_modified) # Update success, set the date to cve_check file. if year == date.today().year: + conn.execute("insert into MTIME values (?)", [pre_update_utc_timestamp.isoformat()]).close() cve_f.write('CVE database update : %s\n\n' % date.today()) conn.commit() diff --git a/meta/recipes-core/meta/cve-update-native.inc b/meta/recipes-core/meta/cve-update-native.inc index 298c89b498..2934f7ad07 100644 --- a/meta/recipes-core/meta/cve-update-native.inc +++ b/meta/recipes-core/meta/cve-update-native.inc @@ -33,6 +33,7 @@ python do_fetch() { import bb.utils import bb.progress import shutil + import time bb.utils.export_proxies(d) @@ -46,26 +47,24 @@ python do_fetch() { # The NVD database changes once a day, so no need to update more frequently # Allow the user to force-update - try: - import time - update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL")) - if update_interval < 0: - bb.note("CVE database update skipped") - if not os.path.exists(db_file): - bb.error("CVE database %s not present, database fetch/update skipped" % db_file) - return - curr_time = time.time() - database_time = os.path.getmtime(db_file) - bb.note("Current time: %s; DB time: %s" % (time.ctime(curr_time), time.ctime(database_time))) - if curr_time < database_time: - bb.warn("Database time is in the future, force DB update") - database_time = 0 - elif curr_time - database_time < update_interval: - bb.note("CVE database recently updated, skipping") - return - - except OSError: - pass + update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL")) + if update_interval < 0: + bb.note("CVE database update skipped") + if not os.path.exists(db_file): + bb.error("CVE database %s not present, database fetch/update skipped" % db_file) + return + + if os.path.exists(db_file): + database_time = get_mtime_timestamp_from(db_file) + if database_time > 0: + curr_time = time.time() + bb.note("Current time: %s; DB time: %s" % (time.ctime(curr_time), time.ctime(database_time))) + if curr_time < database_time: + bb.warn("Database time is in the future, force DB update") + database_time = 0 + elif curr_time - database_time < update_interval: + bb.note("CVE database recently updated, skipping") + return if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): bb.error("BB_NO_NETWORK attempted to disable fetch, this recipe uses CVE_DB_UPDATE_INTERVAL to control download, set to '-1' to disable fetch or update") @@ -107,10 +106,30 @@ def cleanup_db_download(db_tmp_file): os.remove(db_tmp_file) +def get_mtime_timestamp_from(db_file): + """ + Resolve the time when the CVE database was previously updated + """ + import datetime + import sqlite3 + + conn = sqlite3.connect(db_file) + curs = conn.cursor() + res = curs.execute("select TIMESTAMP from MTIME order by TIMESTAMP desc limit 1;") + latest = res.fetchone()[0] + latest = datetime.datetime.strptime(latest, '%Y-%m-%dT%H:%M:%S.%f+00:00') + latest = latest.astimezone(tz=datetime.timezone.utc) + curs.close() + conn.close() + return latest.timestamp() + + def initialize_db(conn): with conn: c = conn.cursor() + c.execute("CREATE TABLE IF NOT EXISTS MTIME (TIMESTAMP INT)") + c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb b/meta/recipes-core/meta/cve-update-nvd2-native.bb index 01d3e8e754..77d7408b16 100644 --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb @@ -72,6 +72,8 @@ def update_db_file(db_tmp_file, d, database_time): import sqlite3 import json + pre_update_utc_timestamp = datetime.datetime.now().astimezone(tz=datetime.timezone.utc) + # Connect to database conn = sqlite3.connect(db_tmp_file) initialize_db(conn) @@ -141,6 +143,7 @@ def update_db_file(db_tmp_file, d, database_time): # Update success, set the date to cve_check file. cve_f.write('CVE database update : %s\n\n' % datetime.date.today()) + conn.execute("insert into MTIME values (?)", [pre_update_utc_timestamp.isoformat()]).close() conn.commit() conn.close()
This should help to avoid problems that will occur if the modification time of database file itself is altered e.g. by unassociated process(es) on the file system which hosts the database file. Since this change updates the database structure by adding a new table, bump the 'minor' version number in database file names to enforce full database fetch. This should also iron out e.g. situation where the database might have inconspicuously omitted entries due to way in which the mtime of database file itself was relied upon. Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> --- meta/classes/cve-check.bbclass | 2 +- .../recipes-core/meta/cve-update-db-native.bb | 3 + meta/recipes-core/meta/cve-update-native.inc | 59 ++++++++++++------- .../meta/cve-update-nvd2-native.bb | 3 + 4 files changed, 46 insertions(+), 21 deletions(-)