[OE-core] [PATCH RFC CFH][sumo 11/47] cve-check: Depends on cve-update-db-native
Mikko Rapeli
mikko.rapeli at bmw.de
Wed Nov 6 15:37:26 UTC 2019
From: Pierre Le Magourou <pierre.lemagourou at softbankrobotics.com>
do_populate_cve_db is a native task.
(From OE-Core rev: 4078da92b49946848cddebe1735f301af161e162)
Signed-off-by: Pierre Le Magourou <pierre.lemagourou at softbankrobotics.com>
Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
Conflicts:
meta/conf/distro/include/maintainers.inc
---
meta/classes/cve-check.bbclass | 2 +-
meta/conf/distro/include/maintainers.inc | 2 +
meta/recipes-core/meta/cve-update-db-native.bb | 143 +++++++++++++++++++++++++
meta/recipes-core/meta/cve-update-db.bb | 143 -------------------------
4 files changed, 146 insertions(+), 144 deletions(-)
create mode 100644 meta/recipes-core/meta/cve-update-db-native.bb
delete mode 100644 meta/recipes-core/meta/cve-update-db.bb
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 81071e3..6ffa0c4 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -63,7 +63,7 @@ python do_cve_check () {
}
addtask cve_check after do_unpack before do_build
-do_cve_check[depends] = "cve-update-db:do_populate_cve_db"
+do_cve_check[depends] = "cve-update-db-native:do_populate_cve_db"
do_cve_check[nostamp] = "1"
python cve_check_cleanup () {
diff --git a/meta/conf/distro/include/maintainers.inc b/meta/conf/distro/include/maintainers.inc
index a299177..ee46a98 100644
--- a/meta/conf/distro/include/maintainers.inc
+++ b/meta/conf/distro/include/maintainers.inc
@@ -122,7 +122,9 @@ RECIPE_MAINTAINER_pn-cryptodev-module = "Robert Yang <liezhi.yang at windriver.com>
RECIPE_MAINTAINER_pn-cryptodev-tests = "Robert Yang <liezhi.yang at windriver.com>"
RECIPE_MAINTAINER_pn-cups = "Chen Qi <Qi.Chen at windriver.com>"
RECIPE_MAINTAINER_pn-curl = "Armin Kuster <akuster at mvista.com>"
+RECIPE_MAINTAINER_pn-cve-check-tool = "Ross Burton <ross.burton at intel.com>"
RECIPE_MAINTAINER_pn-cve-update-db = "Ross Burton <ross.burton at intel.com>"
+RECIPE_MAINTAINER_pn-cve-update-db-native = "Ross Burton <ross.burton at intel.com>"
RECIPE_MAINTAINER_pn-cwautomacros = "Maxin B. John <maxin.john at intel.com>"
RECIPE_MAINTAINER_pn-damageproto = "Armin Kuster <akuster at mvista.com>"
RECIPE_MAINTAINER_pn-db = "Mark Hatle <mark.hatle at windriver.com>"
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
new file mode 100644
index 0000000..ae8f1a9
--- /dev/null
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -0,0 +1,143 @@
+SUMMARY = "Updates the NVD CVE database"
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+PACKAGES = ""
+
+inherit nopackages
+
+deltask do_unpack
+deltask do_patch
+deltask do_configure
+deltask do_compile
+deltask do_install
+deltask do_populate_sysroot
+
+python do_populate_cve_db() {
+ """
+ Update NVD database with json data feed
+ """
+
+ import sqlite3, urllib, shutil, gzip, re
+ from datetime import date
+
+ BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
+ YEAR_START = 2002
+
+ db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
+ db_file = db_dir + '/nvd-json.db'
+ json_tmpfile = db_dir + '/nvd.json.gz'
+ proxy = d.getVar("https_proxy")
+ cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
+
+ if not os.path.isdir(db_dir):
+ os.mkdir(db_dir)
+
+ # Connect to database
+ conn = sqlite3.connect(db_file)
+ c = conn.cursor()
+
+ initialize_db(c)
+
+ for year in range(YEAR_START, date.today().year + 1):
+ year_url = BASE_URL + str(year)
+ meta_url = year_url + ".meta"
+ json_url = year_url + ".json.gz"
+
+ # Retrieve meta last modified date
+ req = urllib.request.Request(meta_url)
+ if proxy:
+ req.set_proxy(proxy, 'https')
+ try:
+ with urllib.request.urlopen(req, timeout=1) as r:
+ date_line = str(r.read().splitlines()[0])
+ last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
+ except:
+ cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
+ break
+
+ # Compare with current db last modified date
+ c.execute("select DATE from META where YEAR = '%d'" % year)
+ meta = c.fetchone()
+ if not meta or meta[0] != last_modified:
+ # Update db with current year json file
+ req = urllib.request.Request(json_url)
+ if proxy:
+ req.set_proxy(proxy, 'https')
+ try:
+ with urllib.request.urlopen(req, timeout=1) as r, \
+ open(json_tmpfile, 'wb') as tmpfile:
+ shutil.copyfileobj(r, tmpfile)
+ except:
+ cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
+ break
+
+ with gzip.open(json_tmpfile, 'rt') as jsonfile:
+ update_db(c, jsonfile)
+ c.execute("insert or replace into META values (?, ?)",
+ [year, last_modified])
+
+ # Update success, set the date to cve_check file.
+ if year == date.today().year:
+ cve_f.write('CVE database update : %s\n\n' % date.today())
+
+ cve_f.close()
+ conn.commit()
+ conn.close()
+}
+
+# DJB2 hash algorithm
+def hash_djb2(s):
+ hash = 5381
+ for x in s:
+ hash = (( hash << 5) + hash) + ord(x)
+
+ return hash & 0xFFFFFFFF
+
+def initialize_db(c):
+ c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
+ c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
+ SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
+ c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID TEXT, \
+ VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
+ c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
+ (PRODUCT, VERSION)")
+
+def update_db(c, json_filename):
+ import json
+ root = json.load(json_filename)
+
+ for elt in root['CVE_Items']:
+ if not elt['impact']:
+ continue
+
+ cveId = elt['cve']['CVE_data_meta']['ID']
+ cveDesc = elt['cve']['description']['description_data'][0]['value']
+ date = elt['lastModifiedDate']
+ accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
+ cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
+
+ try:
+ cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
+ except:
+ cvssv3 = 0.0
+
+ c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
+ [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
+
+ for vendor in elt['cve']['affects']['vendor']['vendor_data']:
+ for product in vendor['product']['product_data']:
+ for version in product['version']['version_data']:
+ product_str = cveId+vendor['vendor_name']+product['product_name']+version['version_value']
+ hashstr = hash_djb2(product_str)
+ c.execute("insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?)",
+ [ hashstr, cveId, vendor['vendor_name'],
+ product['product_name'], version['version_value'],
+ version['version_affected']])
+
+
+
+addtask do_populate_cve_db before do_fetch
+do_populate_cve_db[nostamp] = "1"
+
+EXCLUDE_FROM_WORLD = "1"
diff --git a/meta/recipes-core/meta/cve-update-db.bb b/meta/recipes-core/meta/cve-update-db.bb
deleted file mode 100644
index ae8f1a9..0000000
--- a/meta/recipes-core/meta/cve-update-db.bb
+++ /dev/null
@@ -1,143 +0,0 @@
-SUMMARY = "Updates the NVD CVE database"
-LICENSE = "MIT"
-
-INHIBIT_DEFAULT_DEPS = "1"
-PACKAGES = ""
-
-inherit nopackages
-
-deltask do_unpack
-deltask do_patch
-deltask do_configure
-deltask do_compile
-deltask do_install
-deltask do_populate_sysroot
-
-python do_populate_cve_db() {
- """
- Update NVD database with json data feed
- """
-
- import sqlite3, urllib, shutil, gzip, re
- from datetime import date
-
- BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
- YEAR_START = 2002
-
- db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
- db_file = db_dir + '/nvd-json.db'
- json_tmpfile = db_dir + '/nvd.json.gz'
- proxy = d.getVar("https_proxy")
- cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
-
- if not os.path.isdir(db_dir):
- os.mkdir(db_dir)
-
- # Connect to database
- conn = sqlite3.connect(db_file)
- c = conn.cursor()
-
- initialize_db(c)
-
- for year in range(YEAR_START, date.today().year + 1):
- year_url = BASE_URL + str(year)
- meta_url = year_url + ".meta"
- json_url = year_url + ".json.gz"
-
- # Retrieve meta last modified date
- req = urllib.request.Request(meta_url)
- if proxy:
- req.set_proxy(proxy, 'https')
- try:
- with urllib.request.urlopen(req, timeout=1) as r:
- date_line = str(r.read().splitlines()[0])
- last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
- except:
- cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
- break
-
- # Compare with current db last modified date
- c.execute("select DATE from META where YEAR = '%d'" % year)
- meta = c.fetchone()
- if not meta or meta[0] != last_modified:
- # Update db with current year json file
- req = urllib.request.Request(json_url)
- if proxy:
- req.set_proxy(proxy, 'https')
- try:
- with urllib.request.urlopen(req, timeout=1) as r, \
- open(json_tmpfile, 'wb') as tmpfile:
- shutil.copyfileobj(r, tmpfile)
- except:
- cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
- break
-
- with gzip.open(json_tmpfile, 'rt') as jsonfile:
- update_db(c, jsonfile)
- c.execute("insert or replace into META values (?, ?)",
- [year, last_modified])
-
- # Update success, set the date to cve_check file.
- if year == date.today().year:
- cve_f.write('CVE database update : %s\n\n' % date.today())
-
- cve_f.close()
- conn.commit()
- conn.close()
-}
-
-# DJB2 hash algorithm
-def hash_djb2(s):
- hash = 5381
- for x in s:
- hash = (( hash << 5) + hash) + ord(x)
-
- return hash & 0xFFFFFFFF
-
-def initialize_db(c):
- c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
- c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
- SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
- c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (HASH INTEGER UNIQUE, ID TEXT, \
- VENDOR TEXT, PRODUCT TEXT, VERSION TEXT, OPERATOR TEXT)")
- c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_IDX ON PRODUCTS \
- (PRODUCT, VERSION)")
-
-def update_db(c, json_filename):
- import json
- root = json.load(json_filename)
-
- for elt in root['CVE_Items']:
- if not elt['impact']:
- continue
-
- cveId = elt['cve']['CVE_data_meta']['ID']
- cveDesc = elt['cve']['description']['description_data'][0]['value']
- date = elt['lastModifiedDate']
- accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
- cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
-
- try:
- cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
- except:
- cvssv3 = 0.0
-
- c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
- [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
-
- for vendor in elt['cve']['affects']['vendor']['vendor_data']:
- for product in vendor['product']['product_data']:
- for version in product['version']['version_data']:
- product_str = cveId+vendor['vendor_name']+product['product_name']+version['version_value']
- hashstr = hash_djb2(product_str)
- c.execute("insert or replace into PRODUCTS values (?, ?, ?, ?, ?, ?)",
- [ hashstr, cveId, vendor['vendor_name'],
- product['product_name'], version['version_value'],
- version['version_affected']])
-
-
-
-addtask do_populate_cve_db before do_fetch
-do_populate_cve_db[nostamp] = "1"
-
-EXCLUDE_FROM_WORLD = "1"
--
1.9.1
More information about the Openembedded-core
mailing list