| Message ID | 20251103143157.315178-3-niko.mauno@vaisala.com |
|---|---|
| State | New |
| Headers | show |
| Series | [1/5] cve-update-nvd2-native: pycodestyle fixes | expand |
This should be commented on by Marta, I'm not sure if she brought this back even though it was not working anymore. In case it's accepted, contribution to yocto-docs should follow. Peter > -----Original Message----- > From: Niko Mauno <niko.mauno@vaisala.com> > Sent: Monday, November 3, 2025 15:32 > To: openembedded-core@lists.openembedded.org > Cc: ross.burton@arm.com; rybczynska@gmail.com; Marko, Peter (FT D EU SK > BFS1) <Peter.Marko@siemens.com>; Niko Mauno <niko.mauno@vaisala.com> > Subject: [PATCH 3/5] cve-update: Drop obsolete NVD1 support > > Since enabling NVD1 as NVD_DB_VERSION nowadays leads to BitBake failure > > WARNING: cve-update-db-native-1.0-r0 do_fetch: Failed to fetch CVE data (HTTP > Error 403: Forbidden) > WARNING: cve-update-db-native-1.0-r0 do_fetch: Host IPs are 172.65.90.26, > 172.65.90.25, 172.65.90.24, 172.65.90.27, 2606:4700:78::90:0:180, > 2606:4700:78::90:0:183, 2606:4700:78::90:0:181, 2606:4700:78::90:0:182 > WARNING: cve-update-db-native-1.0-r0 do_fetch: CVE database update failed > ERROR: cve-update-db-native-1.0-r0 do_unpack: Error executing a python > function in exec_func_python() autogenerated: > > Remove the support for obsolete NVD1. > > Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> > --- > meta/classes/cve-check.bbclass | 8 +- > .../recipes-core/meta/cve-update-db-native.bb | 87 +++---------------- > 2 files changed, 15 insertions(+), 80 deletions(-) > > diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass > index c63ebd56e1..259c699af2 100644 > --- a/meta/classes/cve-check.bbclass > +++ b/meta/classes/cve-check.bbclass > @@ -31,11 +31,11 @@ > CVE_PRODUCT ??= "${BPN}" > CVE_VERSION ??= "${PV}" > > -# Possible database sources: NVD1, NVD2, FKIE > +# Possible database sources: NVD2, FKIE > NVD_DB_VERSION ?= "FKIE" > > # Use different file names for each database source, as they synchronize at > different moments, so may be slightly different > -CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if > d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdcve_1-3.db' if > d.getVar('NVD_DB_VERSION') == 'NVD1' else 'nvdfkie_1-1.db'}" > +CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if > d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-1.db'}" > CVE_CHECK_DB_FETCHER ?= "${@'cve-update-nvd2-native' if > d.getVar('NVD_DB_VERSION') == 'NVD2' else 'cve-update-db-native'}" > CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" > CVE_CHECK_DB_FILE ?= > "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" > @@ -108,8 +108,8 @@ python () { > extend_cve_status(d) > > nvd_database_type = d.getVar("NVD_DB_VERSION") > - if nvd_database_type not in ("NVD1", "NVD2", "FKIE"): > - bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD1, > NVD2, FKIE. Defaulting to NVD2") > + if nvd_database_type not in ("NVD2", "FKIE"): > + bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD2, > FKIE. Defaulting to NVD2") > d.setVar("NVD_DB_VERSION", "NVD2") > } > > diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes- > core/meta/cve-update-db-native.bb > index 3a6dc95580..4423216be5 100644 > --- a/meta/recipes-core/meta/cve-update-db-native.bb > +++ b/meta/recipes-core/meta/cve-update-db-native.bb > @@ -11,7 +11,6 @@ deltask do_compile > deltask do_install > deltask do_populate_sysroot > > -NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" > FKIE_URL ?= "https://github.com/fkie-cad/nvd-json-data- > feeds/releases/latest/download/CVE-" > > # CVE database update interval, in seconds. By default: once a day (23*60*60). > @@ -108,30 +107,12 @@ def cleanup_db_download(db_tmp_file): > if os.path.exists(db_tmp_file): > os.remove(db_tmp_file) > > -def db_file_names(d, year, is_nvd): > - if is_nvd: > - year_url = d.getVar('NVDCVE_URL') + str(year) > - meta_url = year_url + ".meta" > - json_url = year_url + ".json.gz" > - return json_url, meta_url > +def db_file_names(d, year): > year_url = d.getVar('FKIE_URL') + str(year) > meta_url = year_url + ".meta" > json_url = year_url + ".json.xz" > return json_url, meta_url > > -def host_db_name(d, is_nvd): > - if is_nvd: > - return "nvd.nist.gov" > - return "github.com" > - > -def db_decompress(d, data, is_nvd): > - import gzip, lzma > - > - if is_nvd: > - return gzip.decompress(data).decode('utf-8') > - # otherwise > - return lzma.decompress(data) > - > def update_db_file(db_tmp_file, d): > """ > Update the given database file > @@ -139,12 +120,12 @@ def update_db_file(db_tmp_file, d): > import bb.progress > import bb.utils > from datetime import date > + import lzma > import sqlite3 > import urllib > > YEAR_START = 2002 > cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) > - is_nvd = d.getVar("NVD_DB_VERSION") == "NVD1" > > # Connect to database > conn = sqlite3.connect(db_tmp_file) > @@ -155,7 +136,7 @@ def update_db_file(db_tmp_file, d): > for i, year in enumerate(range(YEAR_START, date.today().year + 1)): > bb.note("Updating %d" % year) > ph.update((float(i + 1) / total_years) * 100) > - json_url, meta_url = db_file_names(d, year, is_nvd) > + json_url, meta_url = db_file_names(d, year) > > # Retrieve meta last modified date > try: > @@ -164,7 +145,7 @@ def update_db_file(db_tmp_file, d): > cve_f.write('Warning: CVE db update error, Unable to fetch CVE > data.\n\n') > bb.warn("Failed to fetch CVE data (%s)" % e) > import socket > - result = socket.getaddrinfo(host_db_name(d, is_nvd), 443, > proto=socket.IPPROTO_TCP) > + result = socket.getaddrinfo("github.com", 443, > proto=socket.IPPROTO_TCP) > bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t in result))) > return False > > @@ -192,7 +173,7 @@ def update_db_file(db_tmp_file, d): > try: > response = urllib.request.urlopen(json_url, > timeout=cve_socket_timeout) > if response: > - update_db(d, conn, db_decompress(d, response.read(), is_nvd)) > + update_db(conn, lzma.decompress(response.read())) > conn.execute("insert or replace into META values (?, ?)", [year, > last_modified]).close() > except urllib.error.URLError as e: > cve_f.write('Warning: CVE db update error, CVE data is > outdated.\n\n') > @@ -224,17 +205,14 @@ def initialize_db(conn): > > c.close() > > -def parse_node_and_insert(conn, node, cveId, is_nvd): > +def parse_node_and_insert(conn, node, cveId): > # Parse children node if needed > for child in node.get('children', ()): > - parse_node_and_insert(conn, child, cveId, is_nvd) > + parse_node_and_insert(conn, child, cveId) > > - def cpe_generator(is_nvd): > + def cpe_generator(): > match_string = "cpeMatch" > cpe_string = 'criteria' > - if is_nvd: > - match_string = "cpe_match" > - cpe_string = 'cpe23Uri' > > for cpe in node.get(match_string, ()): > if not cpe['vulnerable']: > @@ -290,44 +268,7 @@ def parse_node_and_insert(conn, node, cveId, is_nvd): > # Save processing by representing as -. > yield [cveId, vendor, product, '-', '', '', ''] > > - conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", > cpe_generator(is_nvd)).close() > - > -def update_db_nvdjson(conn, jsondata): > - import json > - root = json.loads(jsondata) > - > - for elt in root['CVE_Items']: > - if not elt['impact']: > - continue > - > - accessVector = None > - vectorString = None > - cvssv2 = 0.0 > - cvssv3 = 0.0 > - cvssv4 = 0.0 > - cveId = elt['cve']['CVE_data_meta']['ID'] > - cveDesc = elt['cve']['description']['description_data'][0]['value'] > - date = elt['lastModifiedDate'] > - try: > - accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector'] > - vectorString = elt['impact']['baseMetricV2']['cvssV2']['vectorString'] > - cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore'] > - except KeyError: > - cvssv2 = 0.0 > - try: > - accessVector = accessVector or > elt['impact']['baseMetricV3']['cvssV3']['attackVector'] > - vectorString = vectorString or > elt['impact']['baseMetricV3']['cvssV3']['vectorString'] > - cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore'] > - except KeyError: > - accessVector = accessVector or "UNKNOWN" > - cvssv3 = 0.0 > - > - conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?, ?, ?)", > - [cveId, cveDesc, cvssv2, cvssv3, cvssv4, date, accessVector, > vectorString]).close() > - > - configurations = elt['configurations']['nodes'] > - for config in configurations: > - parse_node_and_insert(conn, config, cveId, True) > + conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", > cpe_generator()).close() > > def get_metric_entry(metric): > primaries = [c for c in metric if c['type'] == "Primary"] > @@ -338,7 +279,7 @@ def get_metric_entry(metric): > return secondaries[0] > return None > > -def update_db_fkie(conn, jsondata): > +def update_db(conn, jsondata): > import json > root = json.loads(jsondata) > > @@ -403,13 +344,7 @@ def update_db_fkie(conn, jsondata): > for config in elt['configurations']: > # This is suboptimal as it doesn't handle AND/OR and negate, but is > better than nothing > for node in config.get("nodes") or []: > - parse_node_and_insert(conn, node, cveId, False) > - > -def update_db(d, conn, jsondata): > - if (d.getVar("NVD_DB_VERSION") == "FKIE"): > - return update_db_fkie(conn, jsondata) > - else: > - return update_db_nvdjson(conn, jsondata) > + parse_node_and_insert(conn, node, cveId) > > do_fetch[nostamp] = "1" > > -- > 2.47.3
On 11/3/25 17:26, Marko, Peter wrote: > This should be commented on by Marta, I'm not sure if she brought this back even though it was not working anymore. > In case it's accepted, contribution to yocto-docs should follow. > > Peter > Thanks for pointing this out Peter, I'll make a note to submit also relevant changes to yocto-docs as needed. -Niko >> -----Original Message----- >> From: Niko Mauno <niko.mauno@vaisala.com> >> Sent: Monday, November 3, 2025 15:32 >> To: openembedded-core@lists.openembedded.org >> Cc: ross.burton@arm.com; rybczynska@gmail.com; Marko, Peter (FT D EU SK >> BFS1) <Peter.Marko@siemens.com>; Niko Mauno <niko.mauno@vaisala.com> >> Subject: [PATCH 3/5] cve-update: Drop obsolete NVD1 support >> >> Since enabling NVD1 as NVD_DB_VERSION nowadays leads to BitBake failure >> >> WARNING: cve-update-db-native-1.0-r0 do_fetch: Failed to fetch CVE data (HTTP >> Error 403: Forbidden) >> WARNING: cve-update-db-native-1.0-r0 do_fetch: Host IPs are 172.65.90.26, >> 172.65.90.25, 172.65.90.24, 172.65.90.27, 2606:4700:78::90:0:180, >> 2606:4700:78::90:0:183, 2606:4700:78::90:0:181, 2606:4700:78::90:0:182 >> WARNING: cve-update-db-native-1.0-r0 do_fetch: CVE database update failed >> ERROR: cve-update-db-native-1.0-r0 do_unpack: Error executing a python >> function in exec_func_python() autogenerated: >> >> Remove the support for obsolete NVD1. >> >> Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> >> --- >> meta/classes/cve-check.bbclass | 8 +- >> .../recipes-core/meta/cve-update-db-native.bb | 87 +++---------------- >> 2 files changed, 15 insertions(+), 80 deletions(-) >> >> diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass >> index c63ebd56e1..259c699af2 100644 >> --- a/meta/classes/cve-check.bbclass >> +++ b/meta/classes/cve-check.bbclass >> @@ -31,11 +31,11 @@ >> CVE_PRODUCT ??= "${BPN}" >> CVE_VERSION ??= "${PV}" >> >> -# Possible database sources: NVD1, NVD2, FKIE >> +# Possible database sources: NVD2, FKIE >> NVD_DB_VERSION ?= "FKIE" >> >> # Use different file names for each database source, as they synchronize at >> different moments, so may be slightly different >> -CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdcve_1-3.db' if >> d.getVar('NVD_DB_VERSION') == 'NVD1' else 'nvdfkie_1-1.db'}" >> +CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-1.db'}" >> CVE_CHECK_DB_FETCHER ?= "${@'cve-update-nvd2-native' if >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'cve-update-db-native'}" >> CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" >> CVE_CHECK_DB_FILE ?= >> "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" >> @@ -108,8 +108,8 @@ python () { >> extend_cve_status(d) >> >> nvd_database_type = d.getVar("NVD_DB_VERSION") >> - if nvd_database_type not in ("NVD1", "NVD2", "FKIE"): >> - bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD1, >> NVD2, FKIE. Defaulting to NVD2") >> + if nvd_database_type not in ("NVD2", "FKIE"): >> + bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD2, >> FKIE. Defaulting to NVD2") >> d.setVar("NVD_DB_VERSION", "NVD2") >> } >> >> diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes- >> core/meta/cve-update-db-native.bb >> index 3a6dc95580..4423216be5 100644 >> --- a/meta/recipes-core/meta/cve-update-db-native.bb >> +++ b/meta/recipes-core/meta/cve-update-db-native.bb >> @@ -11,7 +11,6 @@ deltask do_compile >> deltask do_install >> deltask do_populate_sysroot >> >> -NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" >> FKIE_URL ?= "https://github.com/fkie-cad/nvd-json-data- >> feeds/releases/latest/download/CVE-" >> >> # CVE database update interval, in seconds. By default: once a day (23*60*60). >> @@ -108,30 +107,12 @@ def cleanup_db_download(db_tmp_file): >> if os.path.exists(db_tmp_file): >> os.remove(db_tmp_file) >> >> -def db_file_names(d, year, is_nvd): >> - if is_nvd: >> - year_url = d.getVar('NVDCVE_URL') + str(year) >> - meta_url = year_url + ".meta" >> - json_url = year_url + ".json.gz" >> - return json_url, meta_url >> +def db_file_names(d, year): >> year_url = d.getVar('FKIE_URL') + str(year) >> meta_url = year_url + ".meta" >> json_url = year_url + ".json.xz" >> return json_url, meta_url >> >> -def host_db_name(d, is_nvd): >> - if is_nvd: >> - return "nvd.nist.gov" >> - return "github.com" >> - >> -def db_decompress(d, data, is_nvd): >> - import gzip, lzma >> - >> - if is_nvd: >> - return gzip.decompress(data).decode('utf-8') >> - # otherwise >> - return lzma.decompress(data) >> - >> def update_db_file(db_tmp_file, d): >> """ >> Update the given database file >> @@ -139,12 +120,12 @@ def update_db_file(db_tmp_file, d): >> import bb.progress >> import bb.utils >> from datetime import date >> + import lzma >> import sqlite3 >> import urllib >> >> YEAR_START = 2002 >> cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) >> - is_nvd = d.getVar("NVD_DB_VERSION") == "NVD1" >> >> # Connect to database >> conn = sqlite3.connect(db_tmp_file) >> @@ -155,7 +136,7 @@ def update_db_file(db_tmp_file, d): >> for i, year in enumerate(range(YEAR_START, date.today().year + 1)): >> bb.note("Updating %d" % year) >> ph.update((float(i + 1) / total_years) * 100) >> - json_url, meta_url = db_file_names(d, year, is_nvd) >> + json_url, meta_url = db_file_names(d, year) >> >> # Retrieve meta last modified date >> try: >> @@ -164,7 +145,7 @@ def update_db_file(db_tmp_file, d): >> cve_f.write('Warning: CVE db update error, Unable to fetch CVE >> data.\n\n') >> bb.warn("Failed to fetch CVE data (%s)" % e) >> import socket >> - result = socket.getaddrinfo(host_db_name(d, is_nvd), 443, >> proto=socket.IPPROTO_TCP) >> + result = socket.getaddrinfo("github.com", 443, >> proto=socket.IPPROTO_TCP) >> bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t in result))) >> return False >> >> @@ -192,7 +173,7 @@ def update_db_file(db_tmp_file, d): >> try: >> response = urllib.request.urlopen(json_url, >> timeout=cve_socket_timeout) >> if response: >> - update_db(d, conn, db_decompress(d, response.read(), is_nvd)) >> + update_db(conn, lzma.decompress(response.read())) >> conn.execute("insert or replace into META values (?, ?)", [year, >> last_modified]).close() >> except urllib.error.URLError as e: >> cve_f.write('Warning: CVE db update error, CVE data is >> outdated.\n\n') >> @@ -224,17 +205,14 @@ def initialize_db(conn): >> >> c.close() >> >> -def parse_node_and_insert(conn, node, cveId, is_nvd): >> +def parse_node_and_insert(conn, node, cveId): >> # Parse children node if needed >> for child in node.get('children', ()): >> - parse_node_and_insert(conn, child, cveId, is_nvd) >> + parse_node_and_insert(conn, child, cveId) >> >> - def cpe_generator(is_nvd): >> + def cpe_generator(): >> match_string = "cpeMatch" >> cpe_string = 'criteria' >> - if is_nvd: >> - match_string = "cpe_match" >> - cpe_string = 'cpe23Uri' >> >> for cpe in node.get(match_string, ()): >> if not cpe['vulnerable']: >> @@ -290,44 +268,7 @@ def parse_node_and_insert(conn, node, cveId, is_nvd): >> # Save processing by representing as -. >> yield [cveId, vendor, product, '-', '', '', ''] >> >> - conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", >> cpe_generator(is_nvd)).close() >> - >> -def update_db_nvdjson(conn, jsondata): >> - import json >> - root = json.loads(jsondata) >> - >> - for elt in root['CVE_Items']: >> - if not elt['impact']: >> - continue >> - >> - accessVector = None >> - vectorString = None >> - cvssv2 = 0.0 >> - cvssv3 = 0.0 >> - cvssv4 = 0.0 >> - cveId = elt['cve']['CVE_data_meta']['ID'] >> - cveDesc = elt['cve']['description']['description_data'][0]['value'] >> - date = elt['lastModifiedDate'] >> - try: >> - accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector'] >> - vectorString = elt['impact']['baseMetricV2']['cvssV2']['vectorString'] >> - cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore'] >> - except KeyError: >> - cvssv2 = 0.0 >> - try: >> - accessVector = accessVector or >> elt['impact']['baseMetricV3']['cvssV3']['attackVector'] >> - vectorString = vectorString or >> elt['impact']['baseMetricV3']['cvssV3']['vectorString'] >> - cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore'] >> - except KeyError: >> - accessVector = accessVector or "UNKNOWN" >> - cvssv3 = 0.0 >> - >> - conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?, ?, ?)", >> - [cveId, cveDesc, cvssv2, cvssv3, cvssv4, date, accessVector, >> vectorString]).close() >> - >> - configurations = elt['configurations']['nodes'] >> - for config in configurations: >> - parse_node_and_insert(conn, config, cveId, True) >> + conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", >> cpe_generator()).close() >> >> def get_metric_entry(metric): >> primaries = [c for c in metric if c['type'] == "Primary"] >> @@ -338,7 +279,7 @@ def get_metric_entry(metric): >> return secondaries[0] >> return None >> >> -def update_db_fkie(conn, jsondata): >> +def update_db(conn, jsondata): >> import json >> root = json.loads(jsondata) >> >> @@ -403,13 +344,7 @@ def update_db_fkie(conn, jsondata): >> for config in elt['configurations']: >> # This is suboptimal as it doesn't handle AND/OR and negate, but is >> better than nothing >> for node in config.get("nodes") or []: >> - parse_node_and_insert(conn, node, cveId, False) >> - >> -def update_db(d, conn, jsondata): >> - if (d.getVar("NVD_DB_VERSION") == "FKIE"): >> - return update_db_fkie(conn, jsondata) >> - else: >> - return update_db_nvdjson(conn, jsondata) >> + parse_node_and_insert(conn, node, cveId) >> >> do_fetch[nostamp] = "1" >> >> -- >> 2.47.3 >
Hello, Thank you for posting this. After having a look at the current NVD situation, I think that the removal of this file is premature, because of the adding of the 2.0 JSON feeds https://nvd.nist.gov/vuln/data-feeds#divJson20Feeds So for people who want to use the NVD data directly from the source still, I think it makes sense to implement the download of the JSON 2.0 feeds (should be closer to the nvd-native than nvd2-native, and *faster*) and then decommission both nvd1 and the current nvd2 (API based). Kind regards, Marta On Tue, Nov 4, 2025 at 9:38 AM Niko Mauno <niko.mauno@vaisala.com> wrote: > > > On 11/3/25 17:26, Marko, Peter wrote: > > This should be commented on by Marta, I'm not sure if she brought this > back even though it was not working anymore. > > In case it's accepted, contribution to yocto-docs should follow. > > > > Peter > > > > Thanks for pointing this out Peter, I'll make a note to submit also > relevant changes to yocto-docs as needed. > -Niko > > >> -----Original Message----- > >> From: Niko Mauno <niko.mauno@vaisala.com> > >> Sent: Monday, November 3, 2025 15:32 > >> To: openembedded-core@lists.openembedded.org > >> Cc: ross.burton@arm.com; rybczynska@gmail.com; Marko, Peter (FT D EU SK > >> BFS1) <Peter.Marko@siemens.com>; Niko Mauno <niko.mauno@vaisala.com> > >> Subject: [PATCH 3/5] cve-update: Drop obsolete NVD1 support > >> > >> Since enabling NVD1 as NVD_DB_VERSION nowadays leads to BitBake failure > >> > >> WARNING: cve-update-db-native-1.0-r0 do_fetch: Failed to fetch CVE > data (HTTP > >> Error 403: Forbidden) > >> WARNING: cve-update-db-native-1.0-r0 do_fetch: Host IPs are > 172.65.90.26, > >> 172.65.90.25, 172.65.90.24, 172.65.90.27, 2606:4700:78::90:0:180, > >> 2606:4700:78::90:0:183, 2606:4700:78::90:0:181, 2606:4700:78::90:0:182 > >> WARNING: cve-update-db-native-1.0-r0 do_fetch: CVE database update > failed > >> ERROR: cve-update-db-native-1.0-r0 do_unpack: Error executing a > python > >> function in exec_func_python() autogenerated: > >> > >> Remove the support for obsolete NVD1. > >> > >> Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> > >> --- > >> meta/classes/cve-check.bbclass | 8 +- > >> .../recipes-core/meta/cve-update-db-native.bb | 87 > +++---------------- > >> 2 files changed, 15 insertions(+), 80 deletions(-) > >> > >> diff --git a/meta/classes/cve-check.bbclass > b/meta/classes/cve-check.bbclass > >> index c63ebd56e1..259c699af2 100644 > >> --- a/meta/classes/cve-check.bbclass > >> +++ b/meta/classes/cve-check.bbclass > >> @@ -31,11 +31,11 @@ > >> CVE_PRODUCT ??= "${BPN}" > >> CVE_VERSION ??= "${PV}" > >> > >> -# Possible database sources: NVD1, NVD2, FKIE > >> +# Possible database sources: NVD2, FKIE > >> NVD_DB_VERSION ?= "FKIE" > >> > >> # Use different file names for each database source, as they > synchronize at > >> different moments, so may be slightly different > >> -CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if > >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdcve_1-3.db' if > >> d.getVar('NVD_DB_VERSION') == 'NVD1' else 'nvdfkie_1-1.db'}" > >> +CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if > >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-1.db'}" > >> CVE_CHECK_DB_FETCHER ?= "${@'cve-update-nvd2-native' if > >> d.getVar('NVD_DB_VERSION') == 'NVD2' else 'cve-update-db-native'}" > >> CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" > >> CVE_CHECK_DB_FILE ?= > >> "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" > >> @@ -108,8 +108,8 @@ python () { > >> extend_cve_status(d) > >> > >> nvd_database_type = d.getVar("NVD_DB_VERSION") > >> - if nvd_database_type not in ("NVD1", "NVD2", "FKIE"): > >> - bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD1, > >> NVD2, FKIE. Defaulting to NVD2") > >> + if nvd_database_type not in ("NVD2", "FKIE"): > >> + bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD2, > >> FKIE. Defaulting to NVD2") > >> d.setVar("NVD_DB_VERSION", "NVD2") > >> } > >> > >> diff --git a/meta/recipes-core/meta/cve-update-db-native.bb > b/meta/recipes- > >> core/meta/cve-update-db-native.bb > >> index 3a6dc95580..4423216be5 100644 > >> --- a/meta/recipes-core/meta/cve-update-db-native.bb > >> +++ b/meta/recipes-core/meta/cve-update-db-native.bb > >> @@ -11,7 +11,6 @@ deltask do_compile > >> deltask do_install > >> deltask do_populate_sysroot > >> > >> -NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" > >> FKIE_URL ?= "https://github.com/fkie-cad/nvd-json-data- > >> feeds/releases/latest/download/CVE-" > >> > >> # CVE database update interval, in seconds. By default: once a day > (23*60*60). > >> @@ -108,30 +107,12 @@ def cleanup_db_download(db_tmp_file): > >> if os.path.exists(db_tmp_file): > >> os.remove(db_tmp_file) > >> > >> -def db_file_names(d, year, is_nvd): > >> - if is_nvd: > >> - year_url = d.getVar('NVDCVE_URL') + str(year) > >> - meta_url = year_url + ".meta" > >> - json_url = year_url + ".json.gz" > >> - return json_url, meta_url > >> +def db_file_names(d, year): > >> year_url = d.getVar('FKIE_URL') + str(year) > >> meta_url = year_url + ".meta" > >> json_url = year_url + ".json.xz" > >> return json_url, meta_url > >> > >> -def host_db_name(d, is_nvd): > >> - if is_nvd: > >> - return "nvd.nist.gov" > >> - return "github.com" > >> - > >> -def db_decompress(d, data, is_nvd): > >> - import gzip, lzma > >> - > >> - if is_nvd: > >> - return gzip.decompress(data).decode('utf-8') > >> - # otherwise > >> - return lzma.decompress(data) > >> - > >> def update_db_file(db_tmp_file, d): > >> """ > >> Update the given database file > >> @@ -139,12 +120,12 @@ def update_db_file(db_tmp_file, d): > >> import bb.progress > >> import bb.utils > >> from datetime import date > >> + import lzma > >> import sqlite3 > >> import urllib > >> > >> YEAR_START = 2002 > >> cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) > >> - is_nvd = d.getVar("NVD_DB_VERSION") == "NVD1" > >> > >> # Connect to database > >> conn = sqlite3.connect(db_tmp_file) > >> @@ -155,7 +136,7 @@ def update_db_file(db_tmp_file, d): > >> for i, year in enumerate(range(YEAR_START, date.today().year > + 1)): > >> bb.note("Updating %d" % year) > >> ph.update((float(i + 1) / total_years) * 100) > >> - json_url, meta_url = db_file_names(d, year, is_nvd) > >> + json_url, meta_url = db_file_names(d, year) > >> > >> # Retrieve meta last modified date > >> try: > >> @@ -164,7 +145,7 @@ def update_db_file(db_tmp_file, d): > >> cve_f.write('Warning: CVE db update error, Unable to > fetch CVE > >> data.\n\n') > >> bb.warn("Failed to fetch CVE data (%s)" % e) > >> import socket > >> - result = socket.getaddrinfo(host_db_name(d, is_nvd), > 443, > >> proto=socket.IPPROTO_TCP) > >> + result = socket.getaddrinfo("github.com", 443, > >> proto=socket.IPPROTO_TCP) > >> bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t > in result))) > >> return False > >> > >> @@ -192,7 +173,7 @@ def update_db_file(db_tmp_file, d): > >> try: > >> response = urllib.request.urlopen(json_url, > >> timeout=cve_socket_timeout) > >> if response: > >> - update_db(d, conn, db_decompress(d, > response.read(), is_nvd)) > >> + update_db(conn, > lzma.decompress(response.read())) > >> conn.execute("insert or replace into META values > (?, ?)", [year, > >> last_modified]).close() > >> except urllib.error.URLError as e: > >> cve_f.write('Warning: CVE db update error, CVE > data is > >> outdated.\n\n') > >> @@ -224,17 +205,14 @@ def initialize_db(conn): > >> > >> c.close() > >> > >> -def parse_node_and_insert(conn, node, cveId, is_nvd): > >> +def parse_node_and_insert(conn, node, cveId): > >> # Parse children node if needed > >> for child in node.get('children', ()): > >> - parse_node_and_insert(conn, child, cveId, is_nvd) > >> + parse_node_and_insert(conn, child, cveId) > >> > >> - def cpe_generator(is_nvd): > >> + def cpe_generator(): > >> match_string = "cpeMatch" > >> cpe_string = 'criteria' > >> - if is_nvd: > >> - match_string = "cpe_match" > >> - cpe_string = 'cpe23Uri' > >> > >> for cpe in node.get(match_string, ()): > >> if not cpe['vulnerable']: > >> @@ -290,44 +268,7 @@ def parse_node_and_insert(conn, node, cveId, > is_nvd): > >> # Save processing by representing as -. > >> yield [cveId, vendor, product, '-', '', '', ''] > >> > >> - conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, > ?)", > >> cpe_generator(is_nvd)).close() > >> - > >> -def update_db_nvdjson(conn, jsondata): > >> - import json > >> - root = json.loads(jsondata) > >> - > >> - for elt in root['CVE_Items']: > >> - if not elt['impact']: > >> - continue > >> - > >> - accessVector = None > >> - vectorString = None > >> - cvssv2 = 0.0 > >> - cvssv3 = 0.0 > >> - cvssv4 = 0.0 > >> - cveId = elt['cve']['CVE_data_meta']['ID'] > >> - cveDesc = > elt['cve']['description']['description_data'][0]['value'] > >> - date = elt['lastModifiedDate'] > >> - try: > >> - accessVector = > elt['impact']['baseMetricV2']['cvssV2']['accessVector'] > >> - vectorString = > elt['impact']['baseMetricV2']['cvssV2']['vectorString'] > >> - cvssv2 = > elt['impact']['baseMetricV2']['cvssV2']['baseScore'] > >> - except KeyError: > >> - cvssv2 = 0.0 > >> - try: > >> - accessVector = accessVector or > >> elt['impact']['baseMetricV3']['cvssV3']['attackVector'] > >> - vectorString = vectorString or > >> elt['impact']['baseMetricV3']['cvssV3']['vectorString'] > >> - cvssv3 = > elt['impact']['baseMetricV3']['cvssV3']['baseScore'] > >> - except KeyError: > >> - accessVector = accessVector or "UNKNOWN" > >> - cvssv3 = 0.0 > >> - > >> - conn.execute("insert or replace into NVD values (?, ?, ?, ?, > ?, ?, ?, ?)", > >> - [cveId, cveDesc, cvssv2, cvssv3, cvssv4, date, > accessVector, > >> vectorString]).close() > >> - > >> - configurations = elt['configurations']['nodes'] > >> - for config in configurations: > >> - parse_node_and_insert(conn, config, cveId, True) > >> + conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, > ?)", > >> cpe_generator()).close() > >> > >> def get_metric_entry(metric): > >> primaries = [c for c in metric if c['type'] == "Primary"] > >> @@ -338,7 +279,7 @@ def get_metric_entry(metric): > >> return secondaries[0] > >> return None > >> > >> -def update_db_fkie(conn, jsondata): > >> +def update_db(conn, jsondata): > >> import json > >> root = json.loads(jsondata) > >> > >> @@ -403,13 +344,7 @@ def update_db_fkie(conn, jsondata): > >> for config in elt['configurations']: > >> # This is suboptimal as it doesn't handle AND/OR and > negate, but is > >> better than nothing > >> for node in config.get("nodes") or []: > >> - parse_node_and_insert(conn, node, cveId, False) > >> - > >> -def update_db(d, conn, jsondata): > >> - if (d.getVar("NVD_DB_VERSION") == "FKIE"): > >> - return update_db_fkie(conn, jsondata) > >> - else: > >> - return update_db_nvdjson(conn, jsondata) > >> + parse_node_and_insert(conn, node, cveId) > >> > >> do_fetch[nostamp] = "1" > >> > >> -- > >> 2.47.3 > > > >
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass index c63ebd56e1..259c699af2 100644 --- a/meta/classes/cve-check.bbclass +++ b/meta/classes/cve-check.bbclass @@ -31,11 +31,11 @@ CVE_PRODUCT ??= "${BPN}" CVE_VERSION ??= "${PV}" -# Possible database sources: NVD1, NVD2, FKIE +# Possible database sources: NVD2, FKIE NVD_DB_VERSION ?= "FKIE" # Use different file names for each database source, as they synchronize at different moments, so may be slightly different -CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdcve_1-3.db' if d.getVar('NVD_DB_VERSION') == 'NVD1' else 'nvdfkie_1-1.db'}" +CVE_CHECK_DB_FILENAME ?= "${@'nvdcve_2-2.db' if d.getVar('NVD_DB_VERSION') == 'NVD2' else 'nvdfkie_1-1.db'}" CVE_CHECK_DB_FETCHER ?= "${@'cve-update-nvd2-native' if d.getVar('NVD_DB_VERSION') == 'NVD2' else 'cve-update-db-native'}" CVE_CHECK_DB_DIR ?= "${STAGING_DIR}/CVE_CHECK" CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/${CVE_CHECK_DB_FILENAME}" @@ -108,8 +108,8 @@ python () { extend_cve_status(d) nvd_database_type = d.getVar("NVD_DB_VERSION") - if nvd_database_type not in ("NVD1", "NVD2", "FKIE"): - bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD1, NVD2, FKIE. Defaulting to NVD2") + if nvd_database_type not in ("NVD2", "FKIE"): + bb.erroronce("Malformed NVD_DB_VERSION, must be one of: NVD2, FKIE. Defaulting to NVD2") d.setVar("NVD_DB_VERSION", "NVD2") } diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb index 3a6dc95580..4423216be5 100644 --- a/meta/recipes-core/meta/cve-update-db-native.bb +++ b/meta/recipes-core/meta/cve-update-db-native.bb @@ -11,7 +11,6 @@ deltask do_compile deltask do_install deltask do_populate_sysroot -NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" FKIE_URL ?= "https://github.com/fkie-cad/nvd-json-data-feeds/releases/latest/download/CVE-" # CVE database update interval, in seconds. By default: once a day (23*60*60). @@ -108,30 +107,12 @@ def cleanup_db_download(db_tmp_file): if os.path.exists(db_tmp_file): os.remove(db_tmp_file) -def db_file_names(d, year, is_nvd): - if is_nvd: - year_url = d.getVar('NVDCVE_URL') + str(year) - meta_url = year_url + ".meta" - json_url = year_url + ".json.gz" - return json_url, meta_url +def db_file_names(d, year): year_url = d.getVar('FKIE_URL') + str(year) meta_url = year_url + ".meta" json_url = year_url + ".json.xz" return json_url, meta_url -def host_db_name(d, is_nvd): - if is_nvd: - return "nvd.nist.gov" - return "github.com" - -def db_decompress(d, data, is_nvd): - import gzip, lzma - - if is_nvd: - return gzip.decompress(data).decode('utf-8') - # otherwise - return lzma.decompress(data) - def update_db_file(db_tmp_file, d): """ Update the given database file @@ -139,12 +120,12 @@ def update_db_file(db_tmp_file, d): import bb.progress import bb.utils from datetime import date + import lzma import sqlite3 import urllib YEAR_START = 2002 cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) - is_nvd = d.getVar("NVD_DB_VERSION") == "NVD1" # Connect to database conn = sqlite3.connect(db_tmp_file) @@ -155,7 +136,7 @@ def update_db_file(db_tmp_file, d): for i, year in enumerate(range(YEAR_START, date.today().year + 1)): bb.note("Updating %d" % year) ph.update((float(i + 1) / total_years) * 100) - json_url, meta_url = db_file_names(d, year, is_nvd) + json_url, meta_url = db_file_names(d, year) # Retrieve meta last modified date try: @@ -164,7 +145,7 @@ def update_db_file(db_tmp_file, d): cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n') bb.warn("Failed to fetch CVE data (%s)" % e) import socket - result = socket.getaddrinfo(host_db_name(d, is_nvd), 443, proto=socket.IPPROTO_TCP) + result = socket.getaddrinfo("github.com", 443, proto=socket.IPPROTO_TCP) bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t in result))) return False @@ -192,7 +173,7 @@ def update_db_file(db_tmp_file, d): try: response = urllib.request.urlopen(json_url, timeout=cve_socket_timeout) if response: - update_db(d, conn, db_decompress(d, response.read(), is_nvd)) + update_db(conn, lzma.decompress(response.read())) conn.execute("insert or replace into META values (?, ?)", [year, last_modified]).close() except urllib.error.URLError as e: cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') @@ -224,17 +205,14 @@ def initialize_db(conn): c.close() -def parse_node_and_insert(conn, node, cveId, is_nvd): +def parse_node_and_insert(conn, node, cveId): # Parse children node if needed for child in node.get('children', ()): - parse_node_and_insert(conn, child, cveId, is_nvd) + parse_node_and_insert(conn, child, cveId) - def cpe_generator(is_nvd): + def cpe_generator(): match_string = "cpeMatch" cpe_string = 'criteria' - if is_nvd: - match_string = "cpe_match" - cpe_string = 'cpe23Uri' for cpe in node.get(match_string, ()): if not cpe['vulnerable']: @@ -290,44 +268,7 @@ def parse_node_and_insert(conn, node, cveId, is_nvd): # Save processing by representing as -. yield [cveId, vendor, product, '-', '', '', ''] - conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator(is_nvd)).close() - -def update_db_nvdjson(conn, jsondata): - import json - root = json.loads(jsondata) - - for elt in root['CVE_Items']: - if not elt['impact']: - continue - - accessVector = None - vectorString = None - cvssv2 = 0.0 - cvssv3 = 0.0 - cvssv4 = 0.0 - cveId = elt['cve']['CVE_data_meta']['ID'] - cveDesc = elt['cve']['description']['description_data'][0]['value'] - date = elt['lastModifiedDate'] - try: - accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector'] - vectorString = elt['impact']['baseMetricV2']['cvssV2']['vectorString'] - cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore'] - except KeyError: - cvssv2 = 0.0 - try: - accessVector = accessVector or elt['impact']['baseMetricV3']['cvssV3']['attackVector'] - vectorString = vectorString or elt['impact']['baseMetricV3']['cvssV3']['vectorString'] - cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore'] - except KeyError: - accessVector = accessVector or "UNKNOWN" - cvssv3 = 0.0 - - conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?, ?, ?)", - [cveId, cveDesc, cvssv2, cvssv3, cvssv4, date, accessVector, vectorString]).close() - - configurations = elt['configurations']['nodes'] - for config in configurations: - parse_node_and_insert(conn, config, cveId, True) + conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close() def get_metric_entry(metric): primaries = [c for c in metric if c['type'] == "Primary"] @@ -338,7 +279,7 @@ def get_metric_entry(metric): return secondaries[0] return None -def update_db_fkie(conn, jsondata): +def update_db(conn, jsondata): import json root = json.loads(jsondata) @@ -403,13 +344,7 @@ def update_db_fkie(conn, jsondata): for config in elt['configurations']: # This is suboptimal as it doesn't handle AND/OR and negate, but is better than nothing for node in config.get("nodes") or []: - parse_node_and_insert(conn, node, cveId, False) - -def update_db(d, conn, jsondata): - if (d.getVar("NVD_DB_VERSION") == "FKIE"): - return update_db_fkie(conn, jsondata) - else: - return update_db_nvdjson(conn, jsondata) + parse_node_and_insert(conn, node, cveId) do_fetch[nostamp] = "1"
Since enabling NVD1 as NVD_DB_VERSION nowadays leads to BitBake failure WARNING: cve-update-db-native-1.0-r0 do_fetch: Failed to fetch CVE data (HTTP Error 403: Forbidden) WARNING: cve-update-db-native-1.0-r0 do_fetch: Host IPs are 172.65.90.26, 172.65.90.25, 172.65.90.24, 172.65.90.27, 2606:4700:78::90:0:180, 2606:4700:78::90:0:183, 2606:4700:78::90:0:181, 2606:4700:78::90:0:182 WARNING: cve-update-db-native-1.0-r0 do_fetch: CVE database update failed ERROR: cve-update-db-native-1.0-r0 do_unpack: Error executing a python function in exec_func_python() autogenerated: Remove the support for obsolete NVD1. Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> --- meta/classes/cve-check.bbclass | 8 +- .../recipes-core/meta/cve-update-db-native.bb | 87 +++---------------- 2 files changed, 15 insertions(+), 80 deletions(-)