[feature] Add support for including extra packages via .csv files.

This commit adds a new top-level variable VIGILES_EXTRA_PACKAGES
which can specify one or more CSV files that specify a list(s) of
packages + versions to include in the manifest and subsequent report
of the CVE check.

README.md

@@ -279,6 +279,60 @@ To use an alternate config, or a config in a non-default location, you can speci
 VIGILES_DASHBOARD_CONFIG = "/tools/timesys/dashboard_config"
 ```
 
+
+### Specifying Additional Packages to Check
+
+In some cases, a BSP may want to include packages that are built outside of
+the Bitbake/Yocto process. If this is required, ```VIGILES_EXTRA_PACKAGES```
+may be used to specify one or more CSV (comma-separated-value) files that
+describe the extra packages to include in the CVE check.
+
+For example, one may set this in their local.conf:
+
+```
+VIGILES_EXTRA_PACKAGES = "${HOME}/projects/this-bsp/non-yocto/yocto-extra.csv"
+```
+
+or perhaps:
+
+```
+VIGILES_EXTRA_PACKAGES = " \
+	${HOME}/projects/this-bsp/non-yocto/yocto-extra-boot.csv \
+	${HOME}/projects/this-bsp/non-yocto/yocto-extra-ui.csv   \
+"
+```
+
+##### CSV Format
+
+The CSV files consist of an optional header and the following fields:
+
+* Product - the CPE Name that packages use in CVEs
+* (optional) Version - the version of the package used.
+* (optional) License - the license of the package used
+
+Blank lines and those starting with **#** are ignored.
+
+One example:
+
+```sh
+$ cat yocto-extra.csv
+# Optional Header:
+product,version,license
+
+# Comments start with '#' and are ignored.
+
+avahi, 0.6
+
+# Blank lines are also ignored
+
+bash, 4.0
+bash, 4.1, "GPL 3.0"
+
+busybox
+```
+
+
+
 Maintenance
 ===========
 

classes/vigiles.bbclass

@@ -191,6 +191,85 @@ VIGILES_BACKFILL := "${@' '.join( \
 )}"
 
 
+##
+#   Additional packages can be included in the manifest by setting
+#    'VIGILES_EXTRA_PACKAGES' in local.conf, which is expected to be a list of
+#    .csv files in the form of:
+#       <product>, <version>, [<license>]
+##
+def _get_extra_packages(d):
+    import csv
+    import json
+    from collections import defaultdict
+
+    vgls_extra = d.getVar('VIGILES_EXTRA_PACKAGES') or ''
+    extra_files = oe.utils.squashspaces(vgls_extra).split(' ')
+    if not extra_files:
+        bb.debug(2, "Vigiles: No Extra Packages.")
+        return {}
+
+    bb.debug(1, "Importing Extra Packages from %s" % extra_files)
+
+    additional = {
+        'additional_licenses': defaultdict(str),
+        'additional_packages': defaultdict(dict)
+    }
+
+    extra_rows = []
+    for extra_csv in extra_files:
+        if not os.path.exists(extra_csv):
+            print("Skipping Non-Existent Extras File: %s" % extra_csv)
+            continue
+        try:
+            with open(extra_csv) as csv_in:
+                reader = csv.reader(csv_in)
+                for row in reader:
+                    if not len(row):
+                        continue
+                    if row[0].startswith('#'):
+                        continue
+
+                    pkg = row[0].strip()
+                    if len(row) > 1:
+                        ver = row[1].strip()
+                    else:
+                        ver = ''
+                    if len(row) > 2:
+                        license = row[2].strip()
+                    else:
+                        license = 'unknown'
+                    extra_rows.append([pkg,ver,license])
+        except Exception as e:
+            bb.warn("Vigiles Extras File: %s" % e)
+            return {}
+
+    if not extra_rows:
+        return {}
+
+    # Check for a CSV header of e.g. "package,version,license" and skip it
+    header = extra_rows[0]
+    if header[0].lower() == "package":
+        extra_rows = extra_rows[1:]
+
+    for row in extra_rows:
+        pkg = row[0]
+        ver = row[1]
+        license = row[2]
+        license_key = pkg + ver
+
+        bb.debug(1, "Extra Package: %s, Version: %s, License: %s = %s" %
+                 (pkg, ver, license_key, license))
+
+        pkg_vers = set(additional['additional_packages'].get(pkg, []))
+        if ver:
+            pkg_vers.add(ver)
+
+        additional['additional_packages'][pkg] = sorted(list(pkg_vers))
+        additional['additional_licenses'][license_key] = license
+
+    return additional
+
+
 def vigiles_image_collect(d):
     from datetime import datetime
 
@@ -214,6 +293,7 @@ def vigiles_image_collect(d):
             packages         = tsmeta_read_dictdir_files(d, "cve", pn_list),
             whitelist        = (d.getVar('VIGILES_WHITELIST') or "").split(),
         )
+    dict_out.update(_get_extra_packages(d))
     return dict_out
 
 python do_vigiles_image() {

conf/vigiles.conf

@@ -9,7 +9,7 @@
 #
 ###########################################################
 
-VIGILES_MANIFEST_VERSION = "1.13"
+VIGILES_MANIFEST_VERSION = "1.15"
 
 VIGILES_TIMESTAMP = "${@time.strftime('%Y-%m-%d_%H.%M.%S', time.strptime(d.getVar('DATETIME', True), '%Y%m%d%H%M%S') ) }"
 VIGILES_TIMESTAMP[vardepsexclude] += "DATETIME"
@@ -36,3 +36,9 @@ VIGILES_UBOOT_CONFIG ??= "auto"
 #
 # To modify it, just append to this variable in local.conf
 VIGILES_WHITELIST ?= "CVE-1234-1234"
+
+#   Additional packages can be included in the manifest by setting
+#    'VIGILES_EXTRA_PACKAGES' in local.conf, which is expected to be a list of
+#    .csv files in the form of:
+#       <product>, <version>, [<license>]
+VIGILES_EXTRA_PACKAGES ??= ""