[OE-core] [PATCH RFC CFH][sumo 01/47] cve-update-db: New recipe to update CVE database
Mikko Rapeli
mikko.rapeli at bmw.de
Wed Nov 6 15:37:16 UTC 2019
From: Pierre Le Magourou <pierre.lemagourou at softbankrobotics.com>
cve-check-tool-native do_populate_cve_db task was using deprecated NVD
xml data feeds, cve-update-db uses NVD json data feeds.
Sqlite database schema was updated to take into account CVSSv3 CVE
scores and operator in affected product versions.
A new META table was added to store the last modification date of the
NVD json data feeds.
(From OE-Core rev: 546d14135c50c6a571dfbf3baf6e9b22ce3d58e0)
Signed-off-by: Pierre Le Magourou <pierre.lemagourou at softbankrobotics.com>
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
Conflicts:
meta/conf/distro/include/maintainers.inc
---
meta/conf/distro/include/maintainers.inc | 2 +-
meta/recipes-core/meta/cve-update-db.bb | 121 +++++++++++++++++++++++++++++++
2 files changed, 122 insertions(+), 1 deletion(-)
create mode 100644 meta/recipes-core/meta/cve-update-db.bb
diff --git a/meta/conf/distro/include/maintainers.inc b/meta/conf/distro/include/maintainers.inc
index 48aff95..a299177 100644
--- a/meta/conf/distro/include/maintainers.inc
+++ b/meta/conf/distro/include/maintainers.inc
@@ -122,7 +122,7 @@ RECIPE_MAINTAINER_pn-cryptodev-module = "Robert Yang <liezhi.yang at windriver.com>
RECIPE_MAINTAINER_pn-cryptodev-tests = "Robert Yang <liezhi.yang at windriver.com>"
RECIPE_MAINTAINER_pn-cups = "Chen Qi <Qi.Chen at windriver.com>"
RECIPE_MAINTAINER_pn-curl = "Armin Kuster <akuster at mvista.com>"
-RECIPE_MAINTAINER_pn-cve-check-tool = "Ross Burton <ross.burton at intel.com>"
+RECIPE_MAINTAINER_pn-cve-update-db = "Ross Burton <ross.burton at intel.com>"
RECIPE_MAINTAINER_pn-cwautomacros = "Maxin B. John <maxin.john at intel.com>"
RECIPE_MAINTAINER_pn-damageproto = "Armin Kuster <akuster at mvista.com>"
RECIPE_MAINTAINER_pn-db = "Mark Hatle <mark.hatle at windriver.com>"
diff --git a/meta/recipes-core/meta/cve-update-db.bb b/meta/recipes-core/meta/cve-update-db.bb
new file mode 100644
index 0000000..522fd23
--- /dev/null
+++ b/meta/recipes-core/meta/cve-update-db.bb
@@ -0,0 +1,121 @@
+SUMMARY = "Updates the NVD CVE database"
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+PACKAGES = ""
+
+inherit nopackages
+
+deltask do_fetch
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
+deltask do_populate_sysroot
+
+python do_populate_cve_db() {
+ """
+ Update NVD database with json data feed
+ """
+
+ import sqlite3, urllib3, shutil, gzip, re
+ from datetime import date
+
+ BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
+ YEAR_START = 2002
+ JSON_TMPFILE = d.getVar("CVE_CHECK_DB_DIR") + '/nvd.json.gz'
+
+ # Connect to database
+ db_file = d.getVar("CVE_CHECK_DB_FILE")
+ conn = sqlite3.connect(db_file)
+ c = conn.cursor()
+
+ initialize_db(c)
+
+ http = urllib3.PoolManager()
+
+ for year in range(YEAR_START, date.today().year + 1):
+ year_url = BASE_URL + str(year)
+ meta_url = year_url + ".meta"
+ json_url = year_url + ".json.gz"
+
+ # Retrieve meta last modified date
+ with http.request('GET', meta_url, preload_content=False) as r:
+ date_line = str(r.data.splitlines()[0])
+ last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
+
+ # Compare with current db last modified date
+ c.execute("select DATE from META where YEAR = '%d'" % year)
+ meta = c.fetchone()
+ if not meta or meta[0] != last_modified:
+ # Update db with current year json file
+ with http.request('GET', json_url, preload_content=False) as r, open(JSON_TMPFILE, 'wb') as tmpfile:
+ shutil.copyfileobj(r, tmpfile)
+ with gzip.open(JSON_TMPFILE, 'rt') as jsonfile:
+ update_db(c, jsonfile)
+ c.execute("insert or replace into META values (?, ?)",
+ [year, last_modified])
+
+ conn.commit()
+ conn.close()
+
+ with open(d.getVar("CVE_CHECK_TMP_FILE"), 'a'):
+ os.utime(d.getVar("CVE_CHECK_TMP_FILE"), None)
+}
+
+# DJB2 hash algorithm
+def hash_djb2(s):
+ hash = 5381
+ for x in s:
+ hash = (( hash << 5) + hash) + ord(x)
+
+ return hash & 0xFFFFFFFF
+
+def initialize_db(c):
+ c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
+ c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
+ SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
+ c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID TEXT, \
+ VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
+ c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
+ (PRODUCT, VERSION)")
+
+def update_db(c, json_filename):
+ import json
+ root = json.load(json_filename)
+
+ for elt in root['CVE_Items']:
+ if not elt['impact']:
+ continue
+
+ cveId = elt['cve']['CVE_data_meta']['ID']
+ cveDesc = elt['cve']['description']['description_data'][0]['value']
+ date = elt['lastModifiedDate']
+ accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
+ cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
+
+ try:
+ cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
+ except:
+ cvssv3 = 0.0
+
+ c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
+ [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
+
+ for vendor in elt['cve']['affects']['vendor']['vendor_data']:
+ for product in vendor['product']['product_data']:
+ for version in product['version']['version_data']:
+ product_str = cveId+vendor['vendor_name']+product['product_name']+version['version_value']
+ hashstr = hash_djb2(product_str)
+ c.execute("insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?)",
+ [ hashstr, cveId, vendor['vendor_name'],
+ product['product_name'], version['version_value'],
+ version['version_affected']])
+
+
+
+addtask do_populate_cve_db before do_cve_check
+do_populate_cve_db[nostamp] = "1"
+
+EXCLUDE_FROM_WORLD = "1"
--
1.9.1
More information about the Openembedded-core
mailing list