Support for creating app-only db by ignoring multiple OS sources (#195)

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "appthreat-vulnerability-db"
-version = "5.7.6"
+version = "5.7.7"
 description = "AppThreat's vulnerability database and package search library with a built-in file based storage. OSV, CVE, GitHub, npm are the primary sources of vulnerabilities."
 authors = [
     {name = "Team AppThreat", email = "cloud@appthreat.com"},

vdb/lib/aqua.py

@@ -50,6 +50,49 @@
     "severity_critical": "CRITICAL",
 }
 
+# Some sources are included by default
+DEFAULT_INCLUDE_SOURCE_PATTERNS = ["alpine-unfixed"] if os.getenv("VDB_IGNORE_ALPINE", "") not in ("true", "1") else []
+
+# Some sources are ignored by default
+DEFAULT_IGNORE_SOURCE_PATTERNS = [
+    "alpine",
+    "cwe",
+    "ghsa",
+    "go",
+    "osv",
+    "redhat-cpe",
+    "kevc",
+    "oval",
+    "glad",
+    "mariner",
+]
+
+# Suse is ignored by default for performance reasons
+if os.getenv("VDB_INCLUDE_SUSE", "") not in ("true", "1"):
+    DEFAULT_IGNORE_SOURCE_PATTERNS.append(f"cvrf{os.sep}suse{os.sep}suse")
+
+
+def get_include_source_patterns():
+    """
+    Constructs include patterns by looking up certain distro-specific environment variables
+    """
+    include_paths = DEFAULT_INCLUDE_SOURCE_PATTERNS
+    for name, paths in config.LINUX_DISTRO_VULN_LIST_PATHS.items():
+        if os.getenv(f"VDB_INCLUDE_{name.upper()}", "") in ("true", "1"):
+            include_paths += paths
+    return include_paths
+
+
+def get_ignored_source_patterns():
+    """
+    Constructs ignore patterns by looking up certain distro-specific environment variables
+    """
+    ignore_paths = DEFAULT_IGNORE_SOURCE_PATTERNS
+    for name, paths in config.LINUX_DISTRO_VULN_LIST_PATHS.items():
+        if os.getenv(f"VDB_IGNORE_{name.upper()}", "") in ("true", "1") or os.getenv(f"VDB_EXCLUDE_{name.upper()}", "") in ("true", "1"):
+            ignore_paths += paths
+    return ignore_paths
+
 
 class AquaSource(NvdSource):
     """Aqua CVE source"""
@@ -119,29 +162,19 @@ def convert(self, cve_data):
             return self.nvd_api_to_vuln(cve_data)
         return []
 
-    def is_supported_source(self, zfname):
-        for distro in (
-            "alpine",
-            "cwe",
-            "ghsa",
-            "go",
-            "osv",
-            "redhat-cpe",
-            "kevc",
-            "oval",
-            "glad",
-            "mariner",
-            f"cvrf{os.sep}suse{os.sep}suse",
-        ):
+    @staticmethod
+    def is_supported_source(zfname):
+        # Include has preference over ignore
+        for distro in get_include_source_patterns():
+            if distro in zfname:
+                return True
+        for distro in get_ignored_source_patterns():
             if distro in zfname:
                 return False
-        nvd_start_year = 2018
-        try:
-            nvd_start_year = int(config.nvd_start_year)
-        except Exception:
-            pass
+        nvd_start_year = config.nvd_start_year
         for year in range(1999, nvd_start_year):
-            for pat in (f"CVE-{year}-", f"{os.sep}{year}{os.sep}", f"ALAS-{year}-", f"ALAS2-{year}-", f"openSUSE-SU-{year}-"):
+            for pat in (
+            f"CVE-{year}-", f"{os.sep}{year}{os.sep}", f"ALAS-{year}-", f"ALAS2-{year}-", f"openSUSE-SU-{year}-"):
                 if pat in zfname:
                     return False
         if zfname.endswith(".json"):
@@ -616,8 +649,6 @@ def suse_to_vuln(self, cve_data):
         packages = cve_data.get("ProductTree", {}).get("Relationships", [])
         if not packages or not len(packages) > 0:
             return ret_data
-        cve_id = ""
-        description = ""
         severity = ""
         # Package name has to be extracted from the title :(
         publishedDate = cve_data.get("Tracking", {}).get("InitialReleaseDate", "")
@@ -768,7 +799,6 @@ def debian_to_vuln(self, cve_data):
         header = cve_data.get("Header")
         annotations = cve_data.get("Annotations")
         cve_id = header.get("ID")
-        pkg_name = ""
         cwe_id = ""
         references = []
         description = header.get("Description", "").replace("(", "").replace(")", "")

vdb/lib/config.py

@@ -56,21 +56,29 @@
     "pub": "https://osv-vulnerabilities.storage.googleapis.com/Pub/all.zip",
     "uvi": "https://osv-vulnerabilities.storage.googleapis.com/UVI/all.zip",
     "github": "https://osv-vulnerabilities.storage.googleapis.com/GitHub%20Actions/all.zip",
-    "android": "https://osv-vulnerabilities.storage.googleapis.com/Android/all.zip",
-    "alpine": "https://osv-vulnerabilities.storage.googleapis.com/Alpine/all.zip",
-    "debian": "https://osv-vulnerabilities.storage.googleapis.com/Debian/all.zip",
     "cran": "https://osv-vulnerabilities.storage.googleapis.com/CRAN/all.zip",
-    "almalinux": "https://osv-vulnerabilities.storage.googleapis.com/AlmaLinux/all.zip",
-    "rockylinux": "https://osv-vulnerabilities.storage.googleapis.com/Rocky%20Linux/all.zip",
 }
 
+# Support for disabling individual distro feeds
+if os.getenv("VDB_IGNORE_ALMALINUX", "") not in ("true", "1"):
+    osv_url_dict["almalinux"] = "https://osv-vulnerabilities.storage.googleapis.com/AlmaLinux/all.zip"
+if os.getenv("VDB_IGNORE_ALPINE", "") not in ("true", "1"):
+    osv_url_dict["alpine"] = "https://osv-vulnerabilities.storage.googleapis.com/Alpine/all.zip"
+if os.getenv("VDB_IGNORE_DEBIAN", "") not in ("true", "1"):
+    osv_url_dict["debian"] = "https://osv-vulnerabilities.storage.googleapis.com/Debian/all.zip"
+if os.getenv("VDB_IGNORE_ROCKYLINUX", "") not in ("true", "1"):
+    osv_url_dict["rockylinux"] = "https://osv-vulnerabilities.storage.googleapis.com/Rocky%20Linux/all.zip"
+
 if os.getenv("OSV_INCLUDE_FUZZ"):
     osv_url_dict[
         "linux"
     ] = "https://osv-vulnerabilities.storage.googleapis.com/Linux/all.zip"
     osv_url_dict[
         "oss-fuzz"
     ] = "https://osv-vulnerabilities.storage.googleapis.com/OSS-Fuzz/all.zip"
+    osv_url_dict["android"] = (
+        "https://osv-vulnerabilities.storage.googleapis.com/Android/all.zip",
+    )
 
 aquasec_vuln_list_url = (
     "https://github.com/appthreat/vuln-list/archive/refs/heads/main.zip"
@@ -95,3 +103,22 @@
 
 # Limits size of unpacked data
 max_buffer_size = 200 * 1024 * 1024  # 200 MiB
+
+# This variable can be used to include or exclude distro-specific data
+# export VDB_IGNORE_ALMALINUX=true
+# export VDB_INCLUDE_ALPINE=true
+LINUX_DISTRO_VULN_LIST_PATHS = {
+    "almalinux": ["alma"],
+    "alpine": ["alpine", "alpine-unfixed"],
+    "amazon": ["amazon"],
+    "arch": ["arch-linux"],
+    "chainguard": ["chainguard"],
+    "opensuse": [f"cvrf{os.sep}suse{os.sep}opensuse"],
+    "suse": [f"cvrf{os.sep}suse{os.sep}suse"],
+    "debian": ["debian"],
+    "photon": ["photon"],
+    "redhat": ["redhat"],
+    "rocky": ["rocky"],
+    "ubuntu": ["ubuntu"],
+    "wolfi": ["wolfi"]
+}