-
Notifications
You must be signed in to change notification settings - Fork 67
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add tool to generate rumble json files for detected vulnerabilities (#…
…890) Signed-off-by: Jamon Camisso <jamonation+git@gmail.com>
- Loading branch information
1 parent
9f893e1
commit e78fb6e
Showing
7 changed files
with
794 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
name: Rumble Vulnerability Data | ||
on: | ||
schedule: | ||
- cron: "1 5 * * *" | ||
workflow_dispatch: | ||
push: | ||
branches: [rumble-vulnerability-data] | ||
|
||
env: | ||
PROJECT_ID: "${{ secrets.PROJECT_ID }}" | ||
STORAGE_BUCKET: "${{ secrets.STORAGE_BUCKET }}" | ||
WORKLOAD_IDENTITY_PROVIDER: "${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}" | ||
SERVICE_ACCOUNT: "${{ secrets.GH_ACTION_SERVICE_ACCOUNT }}" | ||
GH_TOKEN: ${{ github.token }} | ||
|
||
defaults: | ||
run: | ||
shell: bash | ||
working-directory: ./tools/rumble | ||
|
||
jobs: | ||
generate-vulnerability-json: | ||
runs-on: ubuntu-latest | ||
|
||
permissions: | ||
contents: write | ||
id-token: write | ||
|
||
steps: | ||
- name: 'Checkout default branch to $GITHUB_WORKSPACE dir' | ||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3 | ||
|
||
- name: Authenticate to Google Cloud | ||
id: auth | ||
uses: google-github-actions/auth@ceee102ec2387dd9e844e01b530ccd4ec87ce955 # v0 | ||
with: | ||
token_format: 'access_token' | ||
project_id: "${{ env.PROJECT_ID }}" | ||
workload_identity_provider: "${{ env.WORKLOAD_IDENTITY_PROVIDER }}" | ||
service_account: "${{ env.SERVICE_ACCOUNT }}" | ||
|
||
- name: Set up Go | ||
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # actions/setup-go@v4 | ||
with: | ||
go-version: '^1.20.0' | ||
|
||
- name: Fetch latest Grype vulnerability database | ||
shell: bash | ||
run: | | ||
curl -s \ | ||
$(curl -s https://toolbox-data.anchore.io/grype/databases/listing.json \ | ||
|jq -r '.available."5" | .[0] .url') -o- \ | ||
|tar xvz | ||
- name: Generate Rumble JSON files | ||
run: go run . |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,160 @@ | ||
/* | ||
Copyright 2023 Chainguard, Inc. | ||
SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"strings" | ||
|
||
"cloud.google.com/go/bigquery" | ||
"golang.org/x/sync/errgroup" | ||
"google.golang.org/api/iterator" | ||
) | ||
|
||
const cveQueryType = "cve" | ||
const scanQueryType = "scan" | ||
|
||
type bqClient struct { | ||
Client *bigquery.Client | ||
Ctx context.Context | ||
} | ||
|
||
type cve struct { | ||
Vulnerability string | ||
} | ||
|
||
type scan struct { | ||
Row int64 | ||
Image string | ||
Scanner string | ||
Scanner_version string | ||
Scanner_db_version string | ||
Time string | ||
Low_cve_cnt int64 | ||
Med_cve_cnt int64 | ||
High_cve_cnt int64 | ||
Crit_cve_cnt int64 | ||
Unknown_cve_cnt int64 | ||
Tot_cve_cnt int64 | ||
Digest string | ||
} | ||
|
||
func NewBqClient() (bqClient, error) { | ||
var b bqClient | ||
var err error | ||
b.Ctx = context.Background() | ||
if b.Client, err = bigquery.NewClient(b.Ctx, "base-image-rumble"); err != nil { | ||
return b, err | ||
} | ||
return b, nil | ||
} | ||
|
||
const allVulnsQuery = ` | ||
SELECT DISTINCT vulnerability | ||
FROM base-image-rumble.rumble.scheduled_vulns | ||
` | ||
|
||
const affectedImagesQuery = ` | ||
SELECT s1.image, s1.time as time, | ||
FROM base-image-rumble.rumble.scheduled_vulns | ||
AS s2 | ||
INNER JOIN base-image-rumble.rumble.scheduled | ||
AS s1 | ||
ON s1.id = s2.scan_id | ||
WHERE s2.vulnerability = ? | ||
GROUP BY s1.time, s1.image | ||
ORDER BY s1.image, s1.time | ||
` | ||
|
||
func (b *bqClient) queryAffectedImages(qr string, vulns []vuln) ([]vuln, error) { | ||
eg := new(errgroup.Group) | ||
eg.SetLimit(50) | ||
for idx, v := range vulns { | ||
vulnerability := v | ||
i := idx | ||
eg.Go(func() error { | ||
fmt.Printf("querying %v\n", vulnerability.Id) | ||
q := b.Client.Query(qr) | ||
q.Parameters = []bigquery.QueryParameter{ | ||
{ | ||
Value: &bigquery.QueryParameterValue{ | ||
Type: bigquery.StandardSQLDataType{ | ||
TypeKind: "STRING", | ||
}, | ||
Value: vulnerability.Id, | ||
}, | ||
}, | ||
} | ||
|
||
it, err := q.Read(b.Ctx) | ||
if err != nil { | ||
return fmt.Errorf("%v", err) | ||
} | ||
|
||
imagesMap := make(map[string]vulnImage) | ||
for { | ||
res := &vulnImage{} | ||
err := it.Next(res) | ||
if err == iterator.Done { | ||
break | ||
} | ||
if err != nil { | ||
return fmt.Errorf("%v", err) | ||
} | ||
img := imagesMap[res.Image] | ||
img.Dates = append(img.Dates, res.Time) | ||
img.Image = res.Image | ||
imagesMap[res.Image] = img | ||
} | ||
|
||
for _, img := range imagesMap { | ||
if strings.Contains(img.Image, "cgr.dev") { | ||
vulns[i].Chainguard = append(vulns[i].Chainguard, img) | ||
} else { | ||
vulns[i].External = append(vulns[i].External, img) | ||
} | ||
} | ||
|
||
return nil | ||
}) | ||
} | ||
|
||
if err := eg.Wait(); err == nil { | ||
fmt.Println("Successfully saved all vulnerabilities.") | ||
} | ||
return vulns, nil | ||
} | ||
|
||
func (b *bqClient) query(qr string, queryType string) ([]interface{}, error) { | ||
q := b.Client.Query(qr) | ||
|
||
it, err := q.Read(b.Ctx) | ||
if err != nil { | ||
return nil, fmt.Errorf("%v", err) | ||
} | ||
|
||
var records []interface{} | ||
for { | ||
var values interface{} | ||
switch queryType { | ||
case scanQueryType: | ||
values = &scan{} | ||
case cveQueryType: | ||
values = &cve{} | ||
} | ||
err := it.Next(values) | ||
if err == iterator.Done { | ||
break | ||
} | ||
if err != nil { | ||
return nil, fmt.Errorf("%v", err) | ||
} | ||
records = append(records, values) | ||
} | ||
|
||
return records, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
module rumble | ||
|
||
go 1.20 | ||
|
||
require ( | ||
cloud.google.com/go/bigquery v1.52.0 | ||
cloud.google.com/go/storage v1.30.1 | ||
github.com/mattn/go-sqlite3 v1.14.17 | ||
golang.org/x/sync v0.2.0 | ||
google.golang.org/api v0.126.0 | ||
) | ||
|
||
require ( | ||
cloud.google.com/go v0.110.2 // indirect | ||
cloud.google.com/go/compute v1.19.3 // indirect | ||
cloud.google.com/go/compute/metadata v0.2.3 // indirect | ||
cloud.google.com/go/iam v1.1.0 // indirect | ||
github.com/andybalholm/brotli v1.0.4 // indirect | ||
github.com/apache/arrow/go/v12 v12.0.0 // indirect | ||
github.com/apache/thrift v0.16.0 // indirect | ||
github.com/goccy/go-json v0.9.11 // indirect | ||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect | ||
github.com/golang/protobuf v1.5.3 // indirect | ||
github.com/golang/snappy v0.0.4 // indirect | ||
github.com/google/flatbuffers v2.0.8+incompatible // indirect | ||
github.com/google/go-cmp v0.5.9 // indirect | ||
github.com/google/s2a-go v0.1.4 // indirect | ||
github.com/google/uuid v1.3.0 // indirect | ||
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect | ||
github.com/googleapis/gax-go/v2 v2.11.0 // indirect | ||
github.com/klauspost/asmfmt v1.3.2 // indirect | ||
github.com/klauspost/compress v1.15.9 // indirect | ||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect | ||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect | ||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect | ||
github.com/pierrec/lz4/v4 v4.1.15 // indirect | ||
github.com/zeebo/xxh3 v1.0.2 // indirect | ||
go.opencensus.io v0.24.0 // indirect | ||
golang.org/x/crypto v0.9.0 // indirect | ||
golang.org/x/mod v0.10.0 // indirect | ||
golang.org/x/net v0.10.0 // indirect | ||
golang.org/x/oauth2 v0.8.0 // indirect | ||
golang.org/x/sys v0.8.0 // indirect | ||
golang.org/x/text v0.9.0 // indirect | ||
golang.org/x/tools v0.9.1 // indirect | ||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect | ||
google.golang.org/appengine v1.6.7 // indirect | ||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect | ||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect | ||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect | ||
google.golang.org/grpc v1.55.0 // indirect | ||
google.golang.org/protobuf v1.30.0 // indirect | ||
) |
Oops, something went wrong.