@@ -1,111 +1,10 @@
-SUMMARY = "Updates the NVD CVE database"
-LICENSE = "MIT"
-
-INHIBIT_DEFAULT_DEPS = "1"
-
-inherit native
-
-deltask do_patch
-deltask do_configure
-deltask do_compile
-deltask do_install
-deltask do_populate_sysroot
+require cve-update-native.inc
FKIE_URL ?= "https://github.com/fkie-cad/nvd-json-data-feeds/releases/latest/download/CVE-"
-# CVE database update interval, in seconds. By default: once a day (23*60*60).
-# Use 0 to force the update
-# Use a negative value to skip the update
-CVE_DB_UPDATE_INTERVAL ?= "82800"
-
# Timeout for blocking socket operations, such as the connection attempt.
CVE_SOCKET_TIMEOUT ?= "60"
-CVE_CHECK_DB_DLDIR_FILE ?= "${DL_DIR}/CVE_CHECK2/${CVE_CHECK_DB_FILENAME}"
-CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock"
-CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp"
-
-python () {
- if not bb.data.inherits_class("cve-check", d):
- raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
-}
-
-python do_fetch() {
- """
- Update NVD database with json data feed
- """
- import bb.utils
- import bb.progress
- import shutil
-
- bb.utils.export_proxies(d)
-
- db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE")
- db_dir = os.path.dirname(db_file)
- db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE")
-
- cleanup_db_download(db_tmp_file)
-
- # The NVD database changes once a day, so no need to update more frequently
- # Allow the user to force-update
- try:
- import time
- update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL"))
- if update_interval < 0:
- bb.note("CVE database update skipped")
- if not os.path.exists(db_file):
- bb.error("CVE database %s not present, database fetch/update skipped" % db_file)
- return
- curr_time = time.time()
- database_time = os.path.getmtime(db_file)
- bb.note("Current time: %s; DB time: %s" % (time.ctime(curr_time), time.ctime(database_time)))
- if curr_time < database_time:
- bb.warn("Database time is in the future, force DB update")
- elif curr_time - database_time < update_interval:
- bb.note("CVE database recently updated, skipping")
- return
-
- except OSError:
- pass
-
- if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
- bb.error("BB_NO_NETWORK attempted to disable fetch, this recipe uses CVE_DB_UPDATE_INTERVAL to control download, set to '-1' to disable fetch or update")
-
- bb.utils.mkdirhier(db_dir)
- bb.utils.mkdirhier(os.path.dirname(db_tmp_file))
- if os.path.exists(db_file):
- shutil.copy2(db_file, db_tmp_file)
-
- if update_db_file(db_tmp_file, d):
- # Update downloaded correctly, can swap files
- shutil.move(db_tmp_file, db_file)
- else:
- # Update failed, do not modify the database
- bb.warn("CVE database update failed")
- os.remove(db_tmp_file)
-}
-
-do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}"
-do_fetch[file-checksums] = ""
-do_fetch[vardeps] = ""
-
-python do_unpack() {
- import shutil
- shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), d.getVar("CVE_CHECK_DB_FILE"))
-}
-do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} ${CVE_CHECK_DB_FILE_LOCK}"
-
-def cleanup_db_download(db_tmp_file):
- """
- Cleanup the download space from possible failed downloads
- """
-
- # Clean-up the temporary file downloads, we can remove both journal
- # and the temporary database
- if os.path.exists("{0}-journal".format(db_tmp_file)):
- os.remove("{0}-journal".format(db_tmp_file))
- if os.path.exists(db_tmp_file):
- os.remove(db_tmp_file)
def db_file_names(d, year):
year_url = d.getVar('FKIE_URL') + str(year)
@@ -113,7 +12,7 @@ def db_file_names(d, year):
json_url = year_url + ".json.xz"
return json_url, meta_url
-def update_db_file(db_tmp_file, d):
+def update_db_file(db_tmp_file, d, *_):
"""
Update the given database file
"""
@@ -189,21 +88,6 @@ def update_db_file(db_tmp_file, d):
conn.close()
return True
-def initialize_db(conn):
- with conn:
- c = conn.cursor()
-
- c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
-
- c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
- SCOREV2 TEXT, SCOREV3 TEXT, SCOREV4 TEXT, MODIFIED INTEGER, VECTOR TEXT, VECTORSTRING TEXT)")
-
- c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
- VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
- VERSION_END TEXT, OPERATOR_END TEXT)")
- c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")
-
- c.close()
def parse_node_and_insert(conn, node, cveId):
# Parse children node if needed
@@ -345,7 +229,3 @@ def update_db(conn, jsondata):
# This is suboptimal as it doesn't handle AND/OR and negate, but is better than nothing
for node in config.get("nodes") or []:
parse_node_and_insert(conn, node, cveId)
-
-do_fetch[nostamp] = "1"
-
-EXCLUDE_FROM_WORLD = "1"
new file mode 100644
@@ -0,0 +1,127 @@
+# Common definitions for CVE database fetching native recipes
+SUMMARY = "Updates the NVD CVE database"
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+
+inherit native
+
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
+deltask do_populate_sysroot
+
+# CVE database update interval, in seconds. By default: once a day (23*60*60).
+# Use 0 to force the update
+# Use a negative value to skip the update
+CVE_DB_UPDATE_INTERVAL ?= "82800"
+
+CVE_CHECK_DB_DLDIR_FILE ?= "${DL_DIR}/CVE_CHECK2/${CVE_CHECK_DB_FILENAME}"
+CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock"
+CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp"
+
+python() {
+ if not bb.data.inherits_class("cve-check", d):
+ raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
+}
+
+python do_fetch() {
+ """
+ Update NVD database
+ """
+ import bb.utils
+ import bb.progress
+ import shutil
+
+ bb.utils.export_proxies(d)
+
+ db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE")
+ db_dir = os.path.dirname(db_file)
+ db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE")
+
+ cleanup_db_download(db_tmp_file)
+ # By default let's update the whole database (since time 0)
+ database_time = 0
+
+ # The NVD database changes once a day, so no need to update more frequently
+ # Allow the user to force-update
+ try:
+ import time
+ update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL"))
+ if update_interval < 0:
+ bb.note("CVE database update skipped")
+ if not os.path.exists(db_file):
+ bb.error("CVE database %s not present, database fetch/update skipped" % db_file)
+ return
+ curr_time = time.time()
+ database_time = os.path.getmtime(db_file)
+ bb.note("Current time: %s; DB time: %s" % (time.ctime(curr_time), time.ctime(database_time)))
+ if curr_time < database_time:
+ bb.warn("Database time is in the future, force DB update")
+ database_time = 0
+ elif curr_time - database_time < update_interval:
+ bb.note("CVE database recently updated, skipping")
+ return
+
+ except OSError:
+ pass
+
+ if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
+ bb.error("BB_NO_NETWORK attempted to disable fetch, this recipe uses CVE_DB_UPDATE_INTERVAL to control download, set to '-1' to disable fetch or update")
+
+ bb.utils.mkdirhier(db_dir)
+ bb.utils.mkdirhier(os.path.dirname(db_tmp_file))
+ if os.path.exists(db_file):
+ shutil.copy2(db_file, db_tmp_file)
+
+ if update_db_file(db_tmp_file, d, database_time):
+ # Update downloaded correctly, can swap files
+ shutil.move(db_tmp_file, db_file)
+ else:
+ # Update failed, do not modify the database
+ bb.warn("CVE database update failed")
+ os.remove(db_tmp_file)
+}
+do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}"
+do_fetch[file-checksums] = ""
+do_fetch[vardeps] = ""
+do_fetch[nostamp] = "1"
+
+python do_unpack() {
+ import shutil
+ shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), d.getVar("CVE_CHECK_DB_FILE"))
+}
+do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} ${CVE_CHECK_DB_FILE_LOCK}"
+
+def cleanup_db_download(db_tmp_file):
+ """
+ Cleanup the download space from possible failed downloads
+ """
+
+ # Clean-up the temporary file downloads, we can remove both journal
+ # and the temporary database
+ if os.path.exists("{0}-journal".format(db_tmp_file)):
+ os.remove("{0}-journal".format(db_tmp_file))
+ if os.path.exists(db_tmp_file):
+ os.remove(db_tmp_file)
+
+
+def initialize_db(conn):
+ with conn:
+ c = conn.cursor()
+
+ c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
+
+ c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
+ SCOREV2 TEXT, SCOREV3 TEXT, SCOREV4 TEXT, MODIFIED INTEGER, VECTOR TEXT, VECTORSTRING TEXT)")
+
+ c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
+ VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
+ VERSION_END TEXT, OPERATOR_END TEXT)")
+ c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")
+
+ c.close()
+
+
+EXCLUDE_FROM_WORLD = "1"
@@ -1,18 +1,7 @@
-SUMMARY = "Updates the NVD CVE database"
-LICENSE = "MIT"
-
# Important note:
# This product uses the NVD API but is not endorsed or certified by the NVD.
-INHIBIT_DEFAULT_DEPS = "1"
-
-inherit native
-
-deltask do_patch
-deltask do_configure
-deltask do_compile
-deltask do_install
-deltask do_populate_sysroot
+require cve-update-native.inc
NVDCVE_URL ?= "https://services.nvd.nist.gov/rest/json/cves/2.0"
@@ -20,11 +9,6 @@ NVDCVE_URL ?= "https://services.nvd.nist.gov/rest/json/cves/2.0"
# then setting this to get higher rate limits.
NVDCVE_API_KEY ?= ""
-# CVE database update interval, in seconds. By default: once a day (23*60*60).
-# Use 0 to force the update
-# Use a negative value to skip the update
-CVE_DB_UPDATE_INTERVAL ?= "82800"
-
# CVE database incremental update age threshold, in seconds. If the database is
# older than this threshold, do a full re-download, else, do an incremental
# update. By default: the maximum allowed value from NVD: 120 days (120*24*60*60)
@@ -34,95 +18,6 @@ CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000"
# Number of attempts for each http query to nvd server before giving up
CVE_DB_UPDATE_ATTEMPTS ?= "5"
-CVE_CHECK_DB_DLDIR_FILE ?= "${DL_DIR}/CVE_CHECK2/${CVE_CHECK_DB_FILENAME}"
-CVE_CHECK_DB_DLDIR_LOCK ?= "${CVE_CHECK_DB_DLDIR_FILE}.lock"
-CVE_CHECK_DB_TEMP_FILE ?= "${CVE_CHECK_DB_FILE}.tmp"
-
-python () {
- if not bb.data.inherits_class("cve-check", d):
- raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
-}
-
-python do_fetch() {
- """
- Update NVD database with API 2.0
- """
- import bb.utils
- import bb.progress
- import shutil
-
- bb.utils.export_proxies(d)
-
- db_file = d.getVar("CVE_CHECK_DB_DLDIR_FILE")
- db_dir = os.path.dirname(db_file)
- db_tmp_file = d.getVar("CVE_CHECK_DB_TEMP_FILE")
-
- cleanup_db_download(db_tmp_file)
- # By default let's update the whole database (since time 0)
- database_time = 0
-
- # The NVD database changes once a day, so no need to update more frequently
- # Allow the user to force-update
- try:
- import time
- update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL"))
- if update_interval < 0:
- bb.note("CVE database update skipped")
- if not os.path.exists(db_file):
- bb.error("CVE database %s not present, database fetch/update skipped" % db_file)
- return
- curr_time = time.time()
- database_time = os.path.getmtime(db_file)
- bb.note("Current time: %s; DB time: %s" % (time.ctime(curr_time), time.ctime(database_time)))
- if curr_time < database_time:
- bb.warn("Database time is in the future, force DB update")
- database_time = 0
- elif curr_time - database_time < update_interval:
- bb.note("CVE database recently updated, skipping")
- return
-
- except OSError:
- pass
-
- if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
- bb.error("BB_NO_NETWORK attempted to disable fetch, this recipe uses CVE_DB_UPDATE_INTERVAL to control download, set to '-1' to disable fetch or update")
-
- bb.utils.mkdirhier(db_dir)
- bb.utils.mkdirhier(os.path.dirname(db_tmp_file))
- if os.path.exists(db_file):
- shutil.copy2(db_file, db_tmp_file)
-
- if update_db_file(db_tmp_file, d, database_time):
- # Update downloaded correctly, can swap files
- shutil.move(db_tmp_file, db_file)
- else:
- # Update failed, do not modify the database
- bb.warn("CVE database update failed")
- os.remove(db_tmp_file)
-}
-
-do_fetch[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK}"
-do_fetch[file-checksums] = ""
-do_fetch[vardeps] = ""
-
-python do_unpack() {
- import shutil
- shutil.copyfile(d.getVar("CVE_CHECK_DB_DLDIR_FILE"), d.getVar("CVE_CHECK_DB_FILE"))
-}
-do_unpack[lockfiles] += "${CVE_CHECK_DB_DLDIR_LOCK} ${CVE_CHECK_DB_FILE_LOCK}"
-
-def cleanup_db_download(db_tmp_file):
- """
- Cleanup the download space from possible failed downloads
- """
-
- # Clean-up the temporary file downloads, we can remove both journal
- # and the temporary database
- if os.path.exists("{0}-journal".format(db_tmp_file)):
- os.remove("{0}-journal".format(db_tmp_file))
- if os.path.exists(db_tmp_file):
- os.remove(db_tmp_file)
-
def nvd_request_wait(attempt, min_wait):
return min(((2 * attempt) + min_wait), 30)
@@ -251,21 +146,6 @@ def update_db_file(db_tmp_file, d, database_time):
conn.close()
return True
-def initialize_db(conn):
- with conn:
- c = conn.cursor()
-
- c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
-
- c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
- SCOREV2 TEXT, SCOREV3 TEXT, SCOREV4 TEXT, MODIFIED INTEGER, VECTOR TEXT, VECTORSTRING TEXT)")
-
- c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
- VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
- VERSION_END TEXT, OPERATOR_END TEXT)")
- c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")
-
- c.close()
def parse_node_and_insert(conn, node, cveId):
@@ -388,7 +268,3 @@ def update_db(conn, elt):
parse_node_and_insert(conn, node, cveId)
except KeyError:
bb.note("CVE %s has no configurations" % cveId)
-
-do_fetch[nostamp] = "1"
-
-EXCLUDE_FROM_WORLD = "1"
Since there are two recipes for the similar purpose with some considerable differences but also some identical definitions, take a shared inc file into use by relocating common code lines there. Signed-off-by: Niko Mauno <niko.mauno@vaisala.com> --- .../recipes-core/meta/cve-update-db-native.bb | 124 +---------------- meta/recipes-core/meta/cve-update-native.inc | 127 ++++++++++++++++++ .../meta/cve-update-nvd2-native.bb | 126 +---------------- 3 files changed, 130 insertions(+), 247 deletions(-) create mode 100644 meta/recipes-core/meta/cve-update-native.inc