| Message ID | 57de6545695ac11816d670959d9e63666de08e3d.1730950520.git.steve@sakoman.com |
|---|---|
| State | Accepted |
| Delegated to: | Steve Sakoman |
| Headers | show |
| Series | [scarthgap,01/10] cve-check: add support for cvss v4.0 | expand |
Hello, I'm wondering if I'm the only one who couldn't complete do_fetch of cve-update-nvd2-native with maximum retries on CVE API? I tried with or without API Key, from several different machines, I also tried to increase retries number etc, without success. I see on the NVD status page the following warning : "Due to changes described below, a large number of CVE records have recently been updated, resulting in a large increase in API requests. We are aware of the issue and are working to mitigate it. Thank you for your patience." I see the Buildroot switched from NVD to alternative FKIE github database due to that API 2.0 inconsistency :/ I'm not sure what are my options here, quite odd that cybersecurity organisation get impacted by DDoS :D Best regards, Vincent Le jeu. 7 nov. 2024 à 04:38, Steve Sakoman via lists.openembedded.org <steve=sakoman.com@lists.openembedded.org> a écrit : > > From: Richard Purdie <richard.purdie@linuxfoundation.org> > > Rtaher than trying to use a sqlite database over NFS from DL_DIR, work from > a local copy in STAGING DIR after fetching. > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > (cherry picked from commit 03596904392d257572a905a182b92c780d636744) > Signed-off-by: Steve Sakoman <steve@sakoman.com> > --- > meta/classes/cve-check.bbclass | 7 ++++--- > .../meta/cve-update-nvd2-native.bb | 18 +++++++++++++----- > 2 files changed, 17 insertions(+), 8 deletions(-) > > diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass > index d287cf1457..ed219bf472 100644 > --- a/meta/classes/cve-check.bbclass > +++ b/meta/classes/cve-check.bbclass > @@ -31,8 +31,9 @@ > CVE_PRODUCT ??= "${BPN}" > CVE_VERSION ??= "${PV}" > > -CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" > -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2-2.db" > +CVE_CHECK_DB_FILENAME ?= "nvdcve_2-2.db" > +CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" > +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" > CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" > > CVE_CHECK_LOG ?= "${T}/cve.log" > @@ -198,7 +199,7 @@ python do_cve_check () { > } > > addtask cve_check before do_build > -do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" > +do_cve_check[depends] = "cve-update-nvd2-native:do_unpack" > do_cve_check[nostamp] = "1" > > python cve_check_cleanup () { > diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb b/meta/recipes-core/meta/cve-update-nvd2-native.bb > index 92177712dc..5fbe9095cc 100644 > --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb > +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb > @@ -8,7 +8,6 @@ INHIBIT_DEFAULT_DEPS = "1" > > inherit native > > -deltask do_unpack > deltask do_patch > deltask do_configure > deltask do_compile > @@ -35,7 +34,9 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" > # Number of attempts for each http query to nvd server before giving up > CVE_DB_UPDATE_ATTEMPTS ?= "5" > > -CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" > +CVE_CHECK_DB_DLDIR_FILE ?= "${DL_DIR}/CVE_CHECK/${CVE_CHECK_DB_FILENAME}" > +CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock" > +CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp" > > python () { > if not bb.data.inherits_class("cve-check", d): > @@ -52,9 +53,9 @@ python do_fetch() { > > bb.utils.export_proxies(d) > > - db_file = d.getVar("CVE_CHECK_DB_FILE") > + db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE") > db_dir = os.path.dirname(db_file) > - db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") > + db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE") > > cleanup_db_download(db_file, db_tmp_file) > # By default let's update the whole database (since time 0) > @@ -77,6 +78,7 @@ python do_fetch() { > pass > > bb.utils.mkdirhier(db_dir) > + bb.utils.mkdirhier(os.path.dirname(db_tmp_file)) > if os.path.exists(db_file): > shutil.copy2(db_file, db_tmp_file) > > @@ -89,10 +91,16 @@ python do_fetch() { > os.remove(db_tmp_file) > } > > -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" > +do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}" > do_fetch[file-checksums] = "" > do_fetch[vardeps] = "" > > +python do_unpack() { > + import shutil > + shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), d.getVar("CVE_CHECK_DB_FILE")) > +} > +do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} ${CVE_CHECK_DB_FILE_LOCK}" > + > def cleanup_db_download(db_file, db_tmp_file): > """ > Cleanup the download space from possible failed downloads > -- > 2.34.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#206820): https://lists.openembedded.org/g/openembedded-core/message/206820 > Mute This Topic: https://lists.openembedded.org/mt/109438548/3616779 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [vincent.prince.fr@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
Le mar. 26 nov. 2024 à 14:02, Vincent Prince via lists.openembedded.org < vincent.prince.fr=gmail.com@lists.openembedded.org> a écrit : > Hello, > > I'm wondering if I'm the only one who couldn't complete do_fetch of > cve-update-nvd2-native with maximum retries on CVE API? > I tried with or without API Key, from several different machines, I > also tried to increase retries number etc, without success. > I did a successful full download with a crazy retry number (100). It took a long time but it finished eventually. You can monitor the progress with bitbake cve-update-nvd2-native -c fetch -v > I see on the NVD status page the following warning : > "Due to changes described below, a large number of CVE records have > recently been updated, resulting in a large increase in API requests. > We are aware of the issue and are working to mitigate it. Thank you > for your patience." > Thanks for the update, I did not think of checking the website -_-' > I see the Buildroot switched from NVD to alternative FKIE github > database due to that API 2.0 inconsistency :/ > https://github.com/fkie-cad/nvd-json-data-feeds If the situation at NVD does not change we might have to also switch I'm not sure what are my options here, quite odd that cybersecurity > organisation get impacted by DDoS :D > > Best regards, > Vincent > > Le jeu. 7 nov. 2024 à 04:38, Steve Sakoman via lists.openembedded.org > <steve=sakoman.com@lists.openembedded.org> a écrit : > > > > From: Richard Purdie <richard.purdie@linuxfoundation.org> > > > > Rtaher than trying to use a sqlite database over NFS from DL_DIR, work > from > > a local copy in STAGING DIR after fetching. > > > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > > (cherry picked from commit 03596904392d257572a905a182b92c780d636744) > > Signed-off-by: Steve Sakoman <steve@sakoman.com> > > --- > > meta/classes/cve-check.bbclass | 7 ++++--- > > .../meta/cve-update-nvd2-native.bb | 18 +++++++++++++----- > > 2 files changed, 17 insertions(+), 8 deletions(-) > > > > diff --git a/meta/classes/cve-check.bbclass > b/meta/classes/cve-check.bbclass > > index d287cf1457..ed219bf472 100644 > > --- a/meta/classes/cve-check.bbclass > > +++ b/meta/classes/cve-check.bbclass > > @@ -31,8 +31,9 @@ > > CVE_PRODUCT ??= "${BPN}" > > CVE_VERSION ??= "${PV}" > > > > -CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" > > -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2-2.db" > > +CVE_CHECK_DB_FILENAME ?= "nvdcve_2-2.db" > > +CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" > > +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" > > CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" > > > > CVE_CHECK_LOG ?= "${T}/cve.log" > > @@ -198,7 +199,7 @@ python do_cve_check () { > > } > > > > addtask cve_check before do_build > > -do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" > > +do_cve_check[depends] = "cve-update-nvd2-native:do_unpack" > > do_cve_check[nostamp] = "1" > > > > python cve_check_cleanup () { > > diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb > b/meta/recipes-core/meta/cve-update-nvd2-native.bb > > index 92177712dc..5fbe9095cc 100644 > > --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb > > +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb > > @@ -8,7 +8,6 @@ INHIBIT_DEFAULT_DEPS = "1" > > > > inherit native > > > > -deltask do_unpack > > deltask do_patch > > deltask do_configure > > deltask do_compile > > @@ -35,7 +34,9 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" > > # Number of attempts for each http query to nvd server before giving up > > CVE_DB_UPDATE_ATTEMPTS ?= "5" > > > > -CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" > > +CVE_CHECK_DB_DLDIR_FILE ?= > "${DL_DIR}/CVE_CHECK/${CVE_CHECK_DB_FILENAME}" > > +CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock" > > +CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp" > > > > python () { > > if not bb.data.inherits_class("cve-check", d): > > @@ -52,9 +53,9 @@ python do_fetch() { > > > > bb.utils.export_proxies(d) > > > > - db_file = d.getVar("CVE_CHECK_DB_FILE") > > + db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE") > > db_dir = os.path.dirname(db_file) > > - db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") > > + db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE") > > > > cleanup_db_download(db_file, db_tmp_file) > > # By default let's update the whole database (since time 0) > > @@ -77,6 +78,7 @@ python do_fetch() { > > pass > > > > bb.utils.mkdirhier(db_dir) > > + bb.utils.mkdirhier(os.path.dirname(db_tmp_file)) > > if os.path.exists(db_file): > > shutil.copy2(db_file, db_tmp_file) > > > > @@ -89,10 +91,16 @@ python do_fetch() { > > os.remove(db_tmp_file) > > } > > > > -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" > > +do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}" > > do_fetch[file-checksums] = "" > > do_fetch[vardeps] = "" > > > > +python do_unpack() { > > + import shutil > > + shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), > d.getVar("CVE_CHECK_DB_FILE")) > > +} > > +do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} > ${CVE_CHECK_DB_FILE_LOCK}" > > + > > def cleanup_db_download(db_file, db_tmp_file): > > """ > > Cleanup the download space from possible failed downloads > > -- > > 2.34.1 > > > > > > > > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#207851): > https://lists.openembedded.org/g/openembedded-core/message/207851 > Mute This Topic: https://lists.openembedded.org/mt/109438548/4316185 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ > yoann.congal@smile.fr] > -=-=-=-=-=-=-=-=-=-=-=- > >
Le mar. 26 nov. 2024 à 14:18, Yoann Congal <yoann.congal@smile.fr> a écrit : > > > Le mar. 26 nov. 2024 à 14:02, Vincent Prince via lists.openembedded.org < > vincent.prince.fr=gmail.com@lists.openembedded.org> a écrit : > >> Hello, >> >> I'm wondering if I'm the only one who couldn't complete do_fetch of >> cve-update-nvd2-native with maximum retries on CVE API? >> I tried with or without API Key, from several different machines, I >> also tried to increase retries number etc, without success. >> > > I did a successful full download with a crazy retry number (100). It took > a long time but it finished eventually. > > You can monitor the progress with > bitbake cve-update-nvd2-native -c fetch -v > > > >> I see on the NVD status page the following warning : >> "Due to changes described below, a large number of CVE records have >> recently been updated, resulting in a large increase in API requests. >> We are aware of the issue and are working to mitigate it. Thank you >> for your patience." >> > > Thanks for the update, I did not think of checking the website -_-' > > >> I see the Buildroot switched from NVD to alternative FKIE github >> database due to that API 2.0 inconsistency :/ >> > > https://github.com/fkie-cad/nvd-json-data-feeds > > If the situation at NVD does not change we might have to also switch > > I'm not sure what are my options here, quite odd that cybersecurity >> organisation get impacted by DDoS :D >> > This is now tracked in the bugzilla : https://bugzilla.yoctoproject.org/show_bug.cgi?id=15660 > Best regards, >> Vincent >> >> Le jeu. 7 nov. 2024 à 04:38, Steve Sakoman via lists.openembedded.org >> <steve=sakoman.com@lists.openembedded.org> a écrit : >> > >> > From: Richard Purdie <richard.purdie@linuxfoundation.org> >> > >> > Rtaher than trying to use a sqlite database over NFS from DL_DIR, work >> from >> > a local copy in STAGING DIR after fetching. >> > >> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> >> > (cherry picked from commit 03596904392d257572a905a182b92c780d636744) >> > Signed-off-by: Steve Sakoman <steve@sakoman.com> >> > --- >> > meta/classes/cve-check.bbclass | 7 ++++--- >> > .../meta/cve-update-nvd2-native.bb | 18 +++++++++++++----- >> > 2 files changed, 17 insertions(+), 8 deletions(-) >> > >> > diff --git a/meta/classes/cve-check.bbclass >> b/meta/classes/cve-check.bbclass >> > index d287cf1457..ed219bf472 100644 >> > --- a/meta/classes/cve-check.bbclass >> > +++ b/meta/classes/cve-check.bbclass >> > @@ -31,8 +31,9 @@ >> > CVE_PRODUCT ??= "${BPN}" >> > CVE_VERSION ??= "${PV}" >> > >> > -CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" >> > -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2-2.db" >> > +CVE_CHECK_DB_FILENAME ?= "nvdcve_2-2.db" >> > +CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" >> > +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" >> > CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" >> > >> > CVE_CHECK_LOG ?= "${T}/cve.log" >> > @@ -198,7 +199,7 @@ python do_cve_check () { >> > } >> > >> > addtask cve_check before do_build >> > -do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" >> > +do_cve_check[depends] = "cve-update-nvd2-native:do_unpack" >> > do_cve_check[nostamp] = "1" >> > >> > python cve_check_cleanup () { >> > diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb >> b/meta/recipes-core/meta/cve-update-nvd2-native.bb >> > index 92177712dc..5fbe9095cc 100644 >> > --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb >> > +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb >> > @@ -8,7 +8,6 @@ INHIBIT_DEFAULT_DEPS = "1" >> > >> > inherit native >> > >> > -deltask do_unpack >> > deltask do_patch >> > deltask do_configure >> > deltask do_compile >> > @@ -35,7 +34,9 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" >> > # Number of attempts for each http query to nvd server before giving up >> > CVE_DB_UPDATE_ATTEMPTS ?= "5" >> > >> > -CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" >> > +CVE_CHECK_DB_DLDIR_FILE ?= >> "${DL_DIR}/CVE_CHECK/${CVE_CHECK_DB_FILENAME}" >> > +CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock" >> > +CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp" >> > >> > python () { >> > if not bb.data.inherits_class("cve-check", d): >> > @@ -52,9 +53,9 @@ python do_fetch() { >> > >> > bb.utils.export_proxies(d) >> > >> > - db_file = d.getVar("CVE_CHECK_DB_FILE") >> > + db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE") >> > db_dir = os.path.dirname(db_file) >> > - db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") >> > + db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE") >> > >> > cleanup_db_download(db_file, db_tmp_file) >> > # By default let's update the whole database (since time 0) >> > @@ -77,6 +78,7 @@ python do_fetch() { >> > pass >> > >> > bb.utils.mkdirhier(db_dir) >> > + bb.utils.mkdirhier(os.path.dirname(db_tmp_file)) >> > if os.path.exists(db_file): >> > shutil.copy2(db_file, db_tmp_file) >> > >> > @@ -89,10 +91,16 @@ python do_fetch() { >> > os.remove(db_tmp_file) >> > } >> > >> > -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" >> > +do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}" >> > do_fetch[file-checksums] = "" >> > do_fetch[vardeps] = "" >> > >> > +python do_unpack() { >> > + import shutil >> > + shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), >> d.getVar("CVE_CHECK_DB_FILE")) >> > +} >> > +do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} >> ${CVE_CHECK_DB_FILE_LOCK}" >> > + >> > def cleanup_db_download(db_file, db_tmp_file): >> > """ >> > Cleanup the download space from possible failed downloads >> > -- >> > 2.34.1 >> > >> > >> > >> > >> >> -=-=-=-=-=-=-=-=-=-=-=- >> Links: You receive all messages sent to this group. >> View/Reply Online (#207851): >> https://lists.openembedded.org/g/openembedded-core/message/207851 >> Mute This Topic: https://lists.openembedded.org/mt/109438548/4316185 >> Group Owner: openembedded-core+owner@lists.openembedded.org >> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ >> yoann.congal@smile.fr] >> -=-=-=-=-=-=-=-=-=-=-=- >> >> > > -- > Yoann Congal > Smile ECS - Tech expert >
On Tue, Nov 26, 2024 at 2:05 PM Vincent Prince via lists.openembedded.org < vincent.prince.fr=gmail.com@lists.openembedded.org> wrote: > Hello, > > I'm wondering if I'm the only one who couldn't complete do_fetch of > cve-update-nvd2-native with maximum retries on CVE API? > I tried with or without API Key, from several different machines, I > also tried to increase retries number etc, without success. > > I see on the NVD status page the following warning : > "Due to changes described below, a large number of CVE records have > recently been updated, resulting in a large increase in API requests. > We are aware of the issue and are working to mitigate it. Thank you > for your patience." > > I see the Buildroot switched from NVD to alternative FKIE github > database due to that API 2.0 inconsistency :/ > > I'm not sure what are my options here, quite odd that cybersecurity > organisation get impacted by DDoS :D > If it doesn't improve in a reasonable time (like until the end of the week) I'm for mirroring it. The complete image for now. Kind regards, Marta
Le jeu. 28 nov. 2024 à 17:09, Marta Rybczynska via lists.openembedded.org <rybczynska=gmail.com@lists.openembedded.org> a écrit : > On Tue, Nov 26, 2024 at 2:05 PM Vincent Prince via lists.openembedded.org > <vincent.prince.fr=gmail.com@lists.openembedded.org> wrote: > >> Hello, >> >> I'm wondering if I'm the only one who couldn't complete do_fetch of >> cve-update-nvd2-native with maximum retries on CVE API? >> I tried with or without API Key, from several different machines, I >> also tried to increase retries number etc, without success. >> >> I see on the NVD status page the following warning : >> "Due to changes described below, a large number of CVE records have >> recently been updated, resulting in a large increase in API requests. >> We are aware of the issue and are working to mitigate it. Thank you >> for your patience." >> >> I see the Buildroot switched from NVD to alternative FKIE github >> database due to that API 2.0 inconsistency :/ >> >> I'm not sure what are my options here, quite odd that cybersecurity >> organisation get impacted by DDoS :D >> > > If it doesn't improve in a reasonable time (like until the end of the > week) I'm for mirroring it. The complete image for now. > FYI, this looks like it's fixed now : https://valkyrie.yoctoproject.org/#/builders/103 has been green for 2 days and, locally, no HTTP errors. Regards, Kind regards, > Marta > > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#207995): > https://lists.openembedded.org/g/openembedded-core/message/207995 > Mute This Topic: https://lists.openembedded.org/mt/109438548/4316185 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ > yoann.congal@smile.fr] > -=-=-=-=-=-=-=-=-=-=-=- > >
On Sun, Dec 1, 2024 at 2:40 PM Yoann Congal <yoann.congal@smile.fr> wrote: > > > Le jeu. 28 nov. 2024 à 17:09, Marta Rybczynska via lists.openembedded.org > <rybczynska=gmail.com@lists.openembedded.org> a écrit : > >> On Tue, Nov 26, 2024 at 2:05 PM Vincent Prince via lists.openembedded.org >> <vincent.prince.fr=gmail.com@lists.openembedded.org> wrote: >> >>> Hello, >>> >>> I'm wondering if I'm the only one who couldn't complete do_fetch of >>> cve-update-nvd2-native with maximum retries on CVE API? >>> I tried with or without API Key, from several different machines, I >>> also tried to increase retries number etc, without success. >>> >>> I see on the NVD status page the following warning : >>> "Due to changes described below, a large number of CVE records have >>> recently been updated, resulting in a large increase in API requests. >>> We are aware of the issue and are working to mitigate it. Thank you >>> for your patience." >>> >>> I see the Buildroot switched from NVD to alternative FKIE github >>> database due to that API 2.0 inconsistency :/ >>> >>> I'm not sure what are my options here, quite odd that cybersecurity >>> organisation get impacted by DDoS :D >>> >> >> If it doesn't improve in a reasonable time (like until the end of the >> week) I'm for mirroring it. The complete image for now. >> > > FYI, this looks like it's fixed now : > https://valkyrie.yoctoproject.org/#/builders/103 has been green for 2 > days and, locally, no HTTP errors. > > For info, I have a partially working switch to a different source. It isn't complicated, just a question of applying recent modifications to the other fetcher. I will post an RFC so that we do have an alternative the next time it goes down. Kind regards, Marta
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass index d287cf1457..ed219bf472 100644 --- a/meta/classes/cve-check.bbclass +++ b/meta/classes/cve-check.bbclass @@ -31,8 +31,9 @@ CVE_PRODUCT ??= "${BPN}" CVE_VERSION ??= "${PV}" -CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" -CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2-2.db" +CVE_CHECK_DB_FILENAME ?= "nvdcve_2-2.db" +CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" +CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" CVE_CHECK_LOG ?= "${T}/cve.log" @@ -198,7 +199,7 @@ python do_cve_check () { } addtask cve_check before do_build -do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" +do_cve_check[depends] = "cve-update-nvd2-native:do_unpack" do_cve_check[nostamp] = "1" python cve_check_cleanup () { diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb b/meta/recipes-core/meta/cve-update-nvd2-native.bb index 92177712dc..5fbe9095cc 100644 --- a/meta/recipes-core/meta/cve-update-nvd2-native.bb +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb @@ -8,7 +8,6 @@ INHIBIT_DEFAULT_DEPS = "1" inherit native -deltask do_unpack deltask do_patch deltask do_configure deltask do_compile @@ -35,7 +34,9 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" # Number of attempts for each http query to nvd server before giving up CVE_DB_UPDATE_ATTEMPTS ?= "5" -CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" +CVE_CHECK_DB_DLDIR_FILE ?= "${DL_DIR}/CVE_CHECK/${CVE_CHECK_DB_FILENAME}" +CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock" +CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp" python () { if not bb.data.inherits_class("cve-check", d): @@ -52,9 +53,9 @@ python do_fetch() { bb.utils.export_proxies(d) - db_file = d.getVar("CVE_CHECK_DB_FILE") + db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE") db_dir = os.path.dirname(db_file) - db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") + db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE") cleanup_db_download(db_file, db_tmp_file) # By default let's update the whole database (since time 0) @@ -77,6 +78,7 @@ python do_fetch() { pass bb.utils.mkdirhier(db_dir) + bb.utils.mkdirhier(os.path.dirname(db_tmp_file)) if os.path.exists(db_file): shutil.copy2(db_file, db_tmp_file) @@ -89,10 +91,16 @@ python do_fetch() { os.remove(db_tmp_file) } -do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" +do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}" do_fetch[file-checksums] = "" do_fetch[vardeps] = "" +python do_unpack() { + import shutil + shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), d.getVar("CVE_CHECK_DB_FILE")) +} +do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} ${CVE_CHECK_DB_FILE_LOCK}" + def cleanup_db_download(db_file, db_tmp_file): """ Cleanup the download space from possible failed downloads