Add tool to generate rumble json files for detected vulnerabilities (#…

…890)

Signed-off-by: Jamon Camisso <jamonation+git@gmail.com>

.github/workflows/rumble-vulnerability-data.yaml

@@ -0,0 +1,56 @@
+name: Rumble Vulnerability Data
+on:
+  schedule:
+    - cron: "1 5 * * *"
+  workflow_dispatch:
+  push:
+    branches: [rumble-vulnerability-data]
+
+env:
+  PROJECT_ID: "${{ secrets.PROJECT_ID }}"
+  STORAGE_BUCKET: "${{ secrets.STORAGE_BUCKET }}"
+  WORKLOAD_IDENTITY_PROVIDER: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
+  SERVICE_ACCOUNT: "${{ secrets.GH_ACTION_SERVICE_ACCOUNT }}"
+  GH_TOKEN: ${{ github.token }}
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./tools/rumble
+
+jobs:
+  generate-vulnerability-json:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - name: 'Checkout default branch to $GITHUB_WORKSPACE dir'
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@ceee102ec2387dd9e844e01b530ccd4ec87ce955 # v0
+        with:
+          token_format: 'access_token'
+          project_id: "${{ env.PROJECT_ID }}"
+          workload_identity_provider: "${{ env.WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ env.SERVICE_ACCOUNT }}"
+
+      - name: Set up Go
+        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # actions/setup-go@v4
+        with:
+          go-version: '^1.20.0'
+
+      - name: Fetch latest Grype vulnerability database
+        shell: bash
+        run: |
+          curl -s \
+            $(curl -s https://toolbox-data.anchore.io/grype/databases/listing.json \
+            |jq -r '.available."5" | .[0] .url') -o- \
+            |tar xvz
+     
+      - name: Generate Rumble JSON files
+        run: go run .

tools/rumble/bigquery.go

@@ -0,0 +1,160 @@
+/*
+Copyright 2023 Chainguard, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"cloud.google.com/go/bigquery"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/api/iterator"
+)
+
+const cveQueryType = "cve"
+const scanQueryType = "scan"
+
+type bqClient struct {
+	Client *bigquery.Client
+	Ctx    context.Context
+}
+
+type cve struct {
+	Vulnerability string
+}
+
+type scan struct {
+	Row                int64
+	Image              string
+	Scanner            string
+	Scanner_version    string
+	Scanner_db_version string
+	Time               string
+	Low_cve_cnt        int64
+	Med_cve_cnt        int64
+	High_cve_cnt       int64
+	Crit_cve_cnt       int64
+	Unknown_cve_cnt    int64
+	Tot_cve_cnt        int64
+	Digest             string
+}
+
+func NewBqClient() (bqClient, error) {
+	var b bqClient
+	var err error
+	b.Ctx = context.Background()
+	if b.Client, err = bigquery.NewClient(b.Ctx, "base-image-rumble"); err != nil {
+		return b, err
+	}
+	return b, nil
+}
+
+const allVulnsQuery = `
+SELECT DISTINCT vulnerability
+FROM base-image-rumble.rumble.scheduled_vulns
+`
+
+const affectedImagesQuery = `
+SELECT s1.image, s1.time as time,
+FROM base-image-rumble.rumble.scheduled_vulns
+AS s2
+INNER JOIN base-image-rumble.rumble.scheduled
+AS s1
+ON s1.id = s2.scan_id
+WHERE s2.vulnerability = ?
+GROUP BY s1.time, s1.image
+ORDER BY s1.image, s1.time
+`
+
+func (b *bqClient) queryAffectedImages(qr string, vulns []vuln) ([]vuln, error) {
+	eg := new(errgroup.Group)
+	eg.SetLimit(50)
+	for idx, v := range vulns {
+		vulnerability := v
+		i := idx
+		eg.Go(func() error {
+			fmt.Printf("querying %v\n", vulnerability.Id)
+			q := b.Client.Query(qr)
+			q.Parameters = []bigquery.QueryParameter{
+				{
+					Value: &bigquery.QueryParameterValue{
+						Type: bigquery.StandardSQLDataType{
+							TypeKind: "STRING",
+						},
+						Value: vulnerability.Id,
+					},
+				},
+			}
+
+			it, err := q.Read(b.Ctx)
+			if err != nil {
+				return fmt.Errorf("%v", err)
+			}
+
+			imagesMap := make(map[string]vulnImage)
+			for {
+				res := &vulnImage{}
+				err := it.Next(res)
+				if err == iterator.Done {
+					break
+				}
+				if err != nil {
+					return fmt.Errorf("%v", err)
+				}
+				img := imagesMap[res.Image]
+				img.Dates = append(img.Dates, res.Time)
+				img.Image = res.Image
+				imagesMap[res.Image] = img
+			}
+
+			for _, img := range imagesMap {
+				if strings.Contains(img.Image, "cgr.dev") {
+					vulns[i].Chainguard = append(vulns[i].Chainguard, img)
+				} else {
+					vulns[i].External = append(vulns[i].External, img)
+				}
+			}
+
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err == nil {
+		fmt.Println("Successfully saved all vulnerabilities.")
+	}
+	return vulns, nil
+}
+
+func (b *bqClient) query(qr string, queryType string) ([]interface{}, error) {
+	q := b.Client.Query(qr)
+
+	it, err := q.Read(b.Ctx)
+	if err != nil {
+		return nil, fmt.Errorf("%v", err)
+	}
+
+	var records []interface{}
+	for {
+		var values interface{}
+		switch queryType {
+		case scanQueryType:
+			values = &scan{}
+		case cveQueryType:
+			values = &cve{}
+		}
+		err := it.Next(values)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%v", err)
+		}
+		records = append(records, values)
+	}
+
+	return records, nil
+}

tools/rumble/go.mod

@@ -0,0 +1,53 @@
+module rumble
+
+go 1.20
+
+require (
+	cloud.google.com/go/bigquery v1.52.0
+	cloud.google.com/go/storage v1.30.1
+	github.com/mattn/go-sqlite3 v1.14.17
+	golang.org/x/sync v0.2.0
+	google.golang.org/api v0.126.0
+)
+
+require (
+	cloud.google.com/go v0.110.2 // indirect
+	cloud.google.com/go/compute v1.19.3 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v1.1.0 // indirect
+	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/apache/arrow/go/v12 v12.0.0 // indirect
+	github.com/apache/thrift v0.16.0 // indirect
+	github.com/goccy/go-json v0.9.11 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/flatbuffers v2.0.8+incompatible // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/s2a-go v0.1.4 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
+	github.com/klauspost/asmfmt v1.3.2 // indirect
+	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/grpc v1.55.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+)