diff --git a/.github/workflows/rumble-vulnerability-data.yaml b/.github/workflows/rumble-vulnerability-data.yaml index 3e61b66b88..ff5ce372f8 100644 --- a/.github/workflows/rumble-vulnerability-data.yaml +++ b/.github/workflows/rumble-vulnerability-data.yaml @@ -44,13 +44,13 @@ jobs: with: go-version: '^1.20.0' - - name: set up bigqueryrc + - name: Fetch latest Grype vulnerability database shell: bash run: | - # gcloud config set auth/impersonate_service_account "${{ env.SERVICE_ACCOUNT }}" - # the following is just used to quiet the bigqueryrc init message, the query result is unused - # bq query --use_legacy_sql=false --format=csv --max_rows=1 'SELECT COUNT(*) FROM base-image-rumble.rumble.scheduled;' 2>&1 > /dev/null - curl -s $(curl -s https://toolbox-data.anchore.io/grype/databases/listing.json |jq -r '.available."5" | .[0] .url') -o-|tar xvz + curl -s \ + $(curl -s https://toolbox-data.anchore.io/grype/databases/listing.json \ + |jq -r '.available."5" | .[0] .url') -o- \ + |tar xvz - name: Generate Rumble JSON files run: go run . diff --git a/tools/rumble/bigquery.go b/tools/rumble/bigquery.go index 2fb9dbb9d6..591e1b2875 100644 --- a/tools/rumble/bigquery.go +++ b/tools/rumble/bigquery.go @@ -53,53 +53,11 @@ func NewBqClient() (bqClient, error) { return b, nil } -const csvQuery = ` -SELECT -ROW_NUMBER() OVER (ORDER BY time), -image, -scanner, -scanner_version, -scanner_db_version, -FORMAT_DATETIME("%Y-%m-%d %H:%M:%S", DATE(time)) as time, -low_cve_count as low_cve_cnt, -med_cve_count as med_cve_cnt, -high_cve_count as high_cve_cnt, -crit_cve_count as crit_cve_cnt, -unknown_cve_count as unknown_cve_cnt, -low_cve_count + med_cve_count + high_cve_count + crit_cve_count + unknown_cve_count AS tot_cve_cnt, -digest FROM base-image-rumble.rumble.scheduled -WHERE DATE(time) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY) AND CURRENT_DATE() - AND scanner = "grype" - LIMIT 10 -` - const allVulnsQuery = ` SELECT DISTINCT vulnerability FROM base-image-rumble.rumble.scheduled_vulns ` -const cveQuery = ` -WITH ruuuumble AS ( - SELECT s1.image, - s1.time as t, - s1.raw_grype_json, - s2.vulnerability, - s2.installed as version, - s2.type, - s2.severity - FROM base-image-rumble.rumble.scheduled_vulns - AS s2 - INNER JOIN base-image-rumble.rumble.scheduled - AS s1 - ON s1.id = s2.scan_id - WHERE s1.image = "golang:latest" - OR s1.image = "cgr.dev/chainguard/golang:latest" - ) - SELECT image, t, vulnerability, version, type, severity FROM ruuuumble - WHERE DATE(t) BETWEEN DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY) AND CURRENT_DATE() - GROUP BY vulnerability, t, image, version, type, severity -` - const affectedImagesQuery = ` SELECT s1.image, s1.time as time, FROM base-image-rumble.rumble.scheduled_vulns @@ -115,9 +73,10 @@ ORDER BY s1.image, s1.time func (b *bqClient) queryAffectedImages(qr string, vulns []vuln) ([]vuln, error) { eg := new(errgroup.Group) eg.SetLimit(50) - for i, v := range vulns { + for idx, v := range vulns { + vulnerability := v + i := idx eg.Go(func() error { - vulnerability := v fmt.Printf("querying %v\n", vulnerability.Id) q := b.Client.Query(qr) q.Parameters = []bigquery.QueryParameter{ diff --git a/tools/rumble/go.mod b/tools/rumble/go.mod index 3f3e7958f0..c1d95d2da2 100644 --- a/tools/rumble/go.mod +++ b/tools/rumble/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go/bigquery v1.52.0 cloud.google.com/go/storage v1.30.1 github.com/mattn/go-sqlite3 v1.14.17 + golang.org/x/sync v0.2.0 google.golang.org/api v0.126.0 ) @@ -39,7 +40,6 @@ require ( golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.9.1 // indirect @@ -50,4 +50,4 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect -) +) \ No newline at end of file diff --git a/tools/rumble/storage.go b/tools/rumble/storage.go index 4da8fe76d8..430a38bf3f 100644 --- a/tools/rumble/storage.go +++ b/tools/rumble/storage.go @@ -10,12 +10,10 @@ import ( "encoding/csv" "encoding/json" "fmt" - "log" "os" "cloud.google.com/go/storage" "golang.org/x/sync/errgroup" - "google.golang.org/api/iterator" ) type gcsClient struct { @@ -64,10 +62,6 @@ func (g *gcsClient) saveJSON(vulns []vuln) error { return err } - // err = os.WriteFile("/tmp/cves/"+v.Id+".json", js, os.ModePerm) - // if err != nil { - // return err - // } fmt.Printf("Wrote %s\n", fName) return nil }) @@ -79,33 +73,7 @@ func (g *gcsClient) saveJSON(vulns []vuln) error { return nil } -func (c *gcsClient) query() { - bkt := c.Client.Bucket("chainguard-academy") - query := &storage.Query{Prefix: "cve-data"} - - var names [][]string - it := bkt.Objects(c.Ctx, query) - for { - attrs, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - log.Fatal(err) - } - names = append(names, []string{attrs.Name}) - } - - w := csv.NewWriter(os.Stdout) - - for _, v := range names { - if err := w.Write(v); err != nil { - log.Fatalln("error writing record to csv:", err) - } - - } -} - +// unused, but can print a csv to stdout if needed func printRecords(records []interface{}, queryType string) error { w := csv.NewWriter(os.Stdout)