From b60db6ce4128513e4224b6cce70b1cda8d6bf537 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 30 Apr 2020 15:09:34 -0700 Subject: [PATCH 1/6] Draft support for MySQL --- Dockerfile | 3 +- go.mod | 2 + go.sum | 7 ++ pkg/initcache/disk.go | 8 +- pkg/initcache/mysql.go | 164 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 pkg/initcache/mysql.go diff --git a/Dockerfile b/Dockerfile index 7043a21..e74b812 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,10 +16,9 @@ FROM golang WORKDIR /app -# CFG is the path to your configuration file +# CFG is the path to your Triage Party configuration ARG CFG -# Set an env var that matches your github repo name, replace treeder/dockergo here with your repo name ENV SRC_DIR=/src/tparty ENV GO111MODULE=on diff --git a/go.mod b/go.mod index 55b8ab9..23137f2 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,11 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.9.0 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/google/go-github/v31 v31.0.0 github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e github.com/imjasonmiller/godice v0.1.2 + github.com/jmoiron/sqlx v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d gopkg.in/yaml.v2 v2.2.8 diff --git a/go.sum b/go.sum index 820aa08..d59bf8e 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,9 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -19,11 +22,15 @@ github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e h1:0aewS5NT github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/imjasonmiller/godice v0.1.2 h1:T1/sW/HoDzFeuwzOOuQjmeMELz9CzZ53I2CnD+08zD4= github.com/imjasonmiller/godice v0.1.2/go.mod h1:8cTkdnVI+NglU2d6sv+ilYcNaJ5VSTBwvMbFULJd/QQ= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= diff --git a/pkg/initcache/disk.go b/pkg/initcache/disk.go index 35fba65..3ef7c51 100644 --- a/pkg/initcache/disk.go +++ b/pkg/initcache/disk.go @@ -150,5 +150,11 @@ func DefaultDiskPath(configPath string, override string) string { name = name + "_" + strings.Replace(override, "/", "_", -1) } - return filepath.Join(fmt.Sprintf("/var/tmp/tparty_%s.cache", name)) + // os.UserCacheDir() is technically better, but difficult to calculate in Dockerfile + home, err := os.UserHomeDir() + if err != nil { + klog.Exitf("unable to get home directory: %v", err) + } + + return filepath.Join(home, ".tpcache", name) } diff --git a/pkg/initcache/mysql.go b/pkg/initcache/mysql.go new file mode 100644 index 0000000..57ee772 --- /dev/null +++ b/pkg/initcache/mysql.go @@ -0,0 +1,164 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package initcache provides a bootstrap for the in-memory cache + +package initcache + +import ( + "bufio" + "bytes" + "encoding/gob" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/patrickmn/go-cache" + "k8s.io/klog" +) + +const ( + DiskExpireInterval = 65 * 24 * time.Hour + DiskCleanupInterval = 15 * time.Minute +) + +type MySQL struct { + path string + cache *cache.Cache + tableName +} + +// NewMySQL returns a new MySQL cache +func NewMySQL(cfg Config) *Disk { + gob.Register(&Hoard{}) + return &MySQL{path: cfg.Path, tableName: "initcache"} +} + +// Initialize creates or loads the disk cache +func (d *Disk) Initialize() error { + // Make cconnection + + klog.Infof("Initializing with %s ...", d.path) + if err := d.load(); err != nil { + klog.Infof("recreating cache due to load error: %v", err) + return d.create() + } + return nil +} + +func (d *Disk) load() error { + f, err := os.Open(d.path) + if err != nil { + return fmt.Errorf("open: %w", err) + } + defer f.Close() + + decoded := map[string]cache.Item{} + + gd := gob.NewDecoder(bufio.NewReader(f)) + + err = gd.Decode(&decoded) + if err != nil && err != io.EOF { + klog.Errorf("Decode failed: %v", err) + return d.create() + } + + if len(decoded) == 0 { + return fmt.Errorf("no items loaded from disk: %v", decoded) + } + + klog.Infof("%d items loaded from disk", len(decoded)) + d.cache = cache.NewFrom(DiskExpireInterval, DiskCleanupInterval, decoded) + return nil +} + +// Set stores a hoard onto disk +func (d *Disk) Set(key string, h *Hoard) error { + if h.Creation.IsZero() { + h.Creation = time.Now() + } + + d.cache.Set(key, h, DiskExpireInterval) + return nil +} + +// DeleteOlderThan deletes a hoard older than a timestamp +func (d *Disk) DeleteOlderThan(key string, t time.Time) error { + d.cache.Delete(key) + return nil +} + +// GetNewerThan returns a hoard older than a timestamp +func (d *Disk) GetNewerThan(key string, t time.Time) *Hoard { + x, ok := d.cache.Get(key) + if !ok { + klog.V(1).Infof("%s is not in the cache!", key) + return nil + } + + h := x.(*Hoard) + if h.Creation.Before(t) { + klog.V(1).Infof("%s is in cache, but %s is older than %s", key, h.Creation, t) + return nil + } + return h +} + +func (d *Disk) create() error { + klog.Infof("Creating cache, expire interval: %s", DiskExpireInterval) + + d.cache = cache.New(DiskExpireInterval, DiskCleanupInterval) + if err := d.Save(); err != nil { + return fmt.Errorf("save: %w", err) + } + return nil +} + +func (m *MySQL) Save() error { + start := time.Now() + items := d.cache.Items() + + klog.Infof("*** Saving %d items to initcache at %s", len(items), d.path) + defer func() { + klog.Infof("*** mysql.Save took %s", time.Since(start)) + }() + + for k, v := range items { + b := new(bytes.Buffer) + ge := gob.NewEncoder(b) + if err := ge.Encode(v); err != nil { + return fmt.Errorf("encode: %w", err) + } + + _, err := s.db.Exec(fmt.Sprintf(` + REPLACE INTO %s (key, value, ts) + VALUES (?, ?);`, m.tableName), key, b.Bytes(), start) + + if err != nil { + return fmt.Errorf("sql exec: %v (len=%d)", err, len(b)) + } + return nil + } + + // Flush older cache items out + var c string + err := s.db.Get(&c, fmt.Sprintf(`DELETE FROM %s WHERE ts > ?`, m.tableName), ts) + if err != nil { + return err + } +} \ No newline at end of file From d2932320e2ae4628b987281b076d725e6b9fe25c Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 4 May 2020 21:55:30 -0700 Subject: [PATCH 2/6] Persistent cache rewrite, phase 1 --- Dockerfile | 26 +- README.md | 9 +- cmd/server/main.go | 25 +- cmd/tester/main.go | 14 +- examples/minikube-deploy.sh | 6 +- examples/skaffold-deploy.sh | 6 +- go.mod | 4 +- go.sum | 292 ++++++++++++++++++ pkg/hubbub/hubbub.go | 10 +- pkg/hubbub/issue.go | 6 +- pkg/hubbub/match.go | 2 +- pkg/hubbub/orgs.go | 4 +- pkg/hubbub/pull_requests.go | 6 +- pkg/initcache/mysql.go | 164 ---------- pkg/persist/cloudsql.go | 51 +++ pkg/{initcache => persist}/disk.go | 106 +++---- pkg/persist/mem.go | 76 +++++ pkg/persist/mysql.go | 171 ++++++++++ .../initcache.go => persist/persist.go} | 24 +- pkg/triage/triage.go | 4 +- 20 files changed, 715 insertions(+), 291 deletions(-) delete mode 100644 pkg/initcache/mysql.go create mode 100644 pkg/persist/cloudsql.go rename pkg/{initcache => persist}/disk.go (57%) create mode 100644 pkg/persist/mem.go create mode 100644 pkg/persist/mysql.go rename pkg/{initcache/initcache.go => persist/persist.go} (75%) diff --git a/Dockerfile b/Dockerfile index e7d9d8e..cf274a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,32 +13,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang +FROM golang AS builder WORKDIR /app # CFG is the path to your Triage Party configuration ARG CFG +# Build the binary ENV SRC_DIR=/src/tparty ENV GO111MODULE=on - RUN mkdir -p ${SRC_DIR}/cmd ${SRC_DIR}/third_party ${SRC_DIR}/pkg ${SRC_DIR}/site /app/third_party /app/site COPY go.* $SRC_DIR/ COPY cmd ${SRC_DIR}/cmd/ COPY pkg ${SRC_DIR}/pkg/ +WORKDIR $SRC_DIR +RUN go mod download +RUN go build cmd/server/main.go -# Build the binary -RUN cd $SRC_DIR && go mod download -RUN cd $SRC_DIR/cmd/server && go build -o main -RUN cp $SRC_DIR/cmd/server/main /app/ +# Populate disk cache data (optional) +FROM alpine AS persist +ARG CFG +COPY pcache /pc +RUN echo "failure is OK with this next step (cache population)" +RUN mv /pc/$(basename $CFG).pc /config.yaml.pc || touch /config.yaml.pc -# Setup our deployment +# Setup the site data +FROM gcr.io/distroless/base +ARG CFG +COPY --from=builder /src/tparty/main /app/ +COPY --from=persist /config.yaml.pc /app/pcache/config.yaml.pc COPY site /app/site/ COPY third_party /app/third_party/ COPY $CFG /app/config.yaml -# Bad hack: pre-heat the cache in lieu of persistent storage -RUN --mount=type=secret,id=github /app/main --github-token-file=/run/secrets/github --config /app/config.yaml --site /app/site --dry-run - # Run the server at a reasonable refresh rate CMD ["/app/main", "--min-refresh=25s", "--max-refresh=20m", "--config=/app/config.yaml", "--site=/app/site", "--3p=/app/third_party"] diff --git a/README.md b/README.md index 0e29d8f..84c508a 100644 --- a/README.md +++ b/README.md @@ -161,13 +161,8 @@ For full example configurations, see `examples/*.yaml`. There are two that are p Docker: ```shell -env DOCKER_BUILDKIT=1 \ - GITHUB_TOKEN_PATH= \ - docker build --tag=tp \ - --build-arg CFG=examples/generic-project.yaml \ - --secret id=github,src=$GITHUB_TOKEN_PATH . - -docker run -p 8080:8080 tp +docker build --tag=tp --build-arg CFG=examples/generic-project.yaml . +docker run -e GITHUB_TOKEN= -p 8080:8080 tp ``` Cloud Run: diff --git a/cmd/server/main.go b/cmd/server/main.go index da9e717..c386ad2 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -30,7 +30,7 @@ import ( "golang.org/x/oauth2" "k8s.io/klog/v2" - "github.com/google/triage-party/pkg/initcache" + "github.com/google/triage-party/pkg/persist" "github.com/google/triage-party/pkg/site" "github.com/google/triage-party/pkg/triage" "github.com/google/triage-party/pkg/updater" @@ -38,8 +38,10 @@ import ( var ( // shared with tester - configPath = flag.String("config", "", "configuration path") - initCachePath = flag.String("initcache", "", "Where to load the initial cache from (optional)") + configPath = flag.String("config", "", "configuration path") + persistBackend = flag.String("persist-backend", "disk", "Cache persistence backend (disk, mysql, cloudsql)") + persistPath = flag.String("persist-path", "", "Where to persist cache to (automatic)") + reposOverride = flag.String("repos", "", "Override configured repos with this repository (comma separated)") githubTokenFile = flag.String("github-token-file", "", "github token secret file, also settable via GITHUB_TOKEN") @@ -74,15 +76,20 @@ func main() { klog.Exitf("open %s: %v", *configPath, err) } - cachePath := *initCachePath - if cachePath == "" { - cachePath = initcache.DefaultDiskPath(*configPath, *reposOverride) + cachePath := *persistPath + if *persistBackend == "disk" && cachePath == "" { + cachePath = persist.DefaultDiskPath(*configPath, *reposOverride) } + klog.Infof("cache path: %s", cachePath) - c := initcache.New(initcache.Config{Type: "disk", Path: cachePath}) + c := persist.New(persist.Config{ + Type: *persistBackend, + Path: cachePath, + }) + if err := c.Initialize(); err != nil { - klog.Exitf("initcache load to %s: %v", cachePath, err) + klog.Exitf("persist load to %s: %v", cachePath, err) } cfg := triage.Config{ @@ -113,7 +120,7 @@ func main() { // Make sure save works if err := c.Save(); err != nil { - klog.Exitf("initcache save to %s: %v", cachePath, err) + klog.Exitf("persist save to %s: %v", cachePath, err) } u := updater.New(updater.Config{ diff --git a/cmd/tester/main.go b/cmd/tester/main.go index 7620328..a2a8017 100644 --- a/cmd/tester/main.go +++ b/cmd/tester/main.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/google/triage-party/pkg/initcache" + "github.com/google/triage-party/pkg/persist" "github.com/google/triage-party/pkg/triage" "github.com/google/go-github/v31/github" @@ -34,7 +34,7 @@ import ( var ( // shared with tester configPath = flag.String("config", "", "configuration path") - initCachePath = flag.String("initcache", "", "Where to load the initial cache from (optional)") + persistPath = flag.String("persist", "", "Where to load the initial cache from (optional)") reposOverride = flag.String("repos", "", "Override configured repos with this repository (comma separated)") githubTokenFile = flag.String("github-token-file", "", "github token secret file, also settable via GITHUB_TOKEN") @@ -66,14 +66,14 @@ func main() { klog.Exitf("open %s: %v", *configPath, err) } - cachePath := *initCachePath + cachePath := *persistPath if cachePath == "" { - cachePath = initcache.DefaultDiskPath(*configPath, *reposOverride) + cachePath = persist.DefaultDiskPath(*configPath, *reposOverride) } - c := initcache.New(initcache.Config{Type: "disk", Path: cachePath}) + c := persist.New(persist.Config{Type: "disk", Path: cachePath}) if err := c.Initialize(); err != nil { - klog.Exitf("initcache load to %s: %v", cachePath, err) + klog.Exitf("persist load to %s: %v", cachePath, err) } cfg := triage.Config{ @@ -99,7 +99,7 @@ func main() { } if err := c.Save(); err != nil { - klog.Exitf("initcache save to %s: %v", cachePath, err) + klog.Exitf("persist save to %s: %v", cachePath, err) } } diff --git a/examples/minikube-deploy.sh b/examples/minikube-deploy.sh index 989df6f..abd459e 100755 --- a/examples/minikube-deploy.sh +++ b/examples/minikube-deploy.sh @@ -23,15 +23,11 @@ export IMAGE=gcr.io/k8s-minikube/triage-party export SERVICE_NAME=teaparty export CONFIG_FILE=examples/minikube.yaml -env DOCKER_BUILDKIT=1 docker build \ - -t "${IMAGE}" \ - --build-arg "CFG=${CONFIG_FILE}" \ - --secret "id=github,src=${GITHUB_TOKEN_PATH}" . +docker build -t "${IMAGE}" --build-arg "CFG=${CONFIG_FILE}" . docker push "${IMAGE}" || exit 2 readonly token="$(cat ${GITHUB_TOKEN_PATH})" - gcloud beta run deploy "${SERVICE_NAME}" \ --project "${PROJECT}" \ --image "${IMAGE}" \ diff --git a/examples/skaffold-deploy.sh b/examples/skaffold-deploy.sh index 5e03a7a..11494dd 100755 --- a/examples/skaffold-deploy.sh +++ b/examples/skaffold-deploy.sh @@ -23,15 +23,11 @@ export IMAGE="gcr.io/k8s-skaffold/teaparty:$(date +%F-%s)" export SERVICE_NAME=skaffold-triage-party export CONFIG_FILE=examples/skaffold.yaml -env DOCKER_BUILDKIT=1 docker build \ - -t "${IMAGE}" \ - --build-arg "CFG=${CONFIG_FILE}" \ - --secret "id=github,src=${GITHUB_TOKEN_PATH}" . +docker build -t "${IMAGE}" --build-arg "CFG=${CONFIG_FILE}" . docker push "${IMAGE}" || exit 2 readonly token="$(cat ${GITHUB_TOKEN_PATH})" - gcloud beta run deploy "${SERVICE_NAME}" \ --project "${PROJECT}" \ --image "${IMAGE}" \ diff --git a/go.mod b/go.mod index f7bcca1..d2aa00e 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module github.com/google/triage-party go 1.14 require ( + github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20200501161113-5e9e23d7cb91 github.com/davecgh/go-spew v1.1.1 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.9.0 // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/go-sql-driver/mysql v1.5.0 github.com/google/go-github/v31 v31.0.0 github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e github.com/imjasonmiller/godice v0.1.2 @@ -14,5 +15,6 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d gopkg.in/yaml.v2 v2.2.8 + k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.0.0 ) diff --git a/go.sum b/go.sum index c457c11..b9ae939 100644 --- a/go.sum +++ b/go.sum @@ -1,59 +1,351 @@ +bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.56.0 h1:WRz29PgAsVEyPSDHyk+0fpEkwEFyfhHn+JbksT6gIL4= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20200501161113-5e9e23d7cb91 h1:KxsIcqivuZu1VnrQRTSWdKgu/5CeryWzjakR81XSIBs= +github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20200501161113-5e9e23d7cb91/go.mod h1:JaTTAYKXdMsyO5t+knEPNeaonOxMb/+0wYbO0pbiGuo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo= github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e h1:0aewS5NTyxftZHSnFaJmWE5oCCrj4DyEXkAiMa1iZJM= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imjasonmiller/godice v0.1.2 h1:T1/sW/HoDzFeuwzOOuQjmeMELz9CzZ53I2CnD+08zD4= github.com/imjasonmiller/godice v0.1.2/go.mod h1:8cTkdnVI+NglU2d6sv+ilYcNaJ5VSTBwvMbFULJd/QQ= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a h1:y6sBfNd1b9Wy08a6K1Z1DZc4aXABUN5TKjkYhz7UKmo= +golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY= +google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200420144010-e5e8543f8aeb h1:nAFaltAMbNVA0rixtwvdnqgSVLX3HFUUvMkEklmzbYM= +google.golang.org/genproto v0.0.0-20200420144010-e5e8543f8aeb/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= +google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/hubbub/hubbub.go b/pkg/hubbub/hubbub.go index 5569657..7deb585 100644 --- a/pkg/hubbub/hubbub.go +++ b/pkg/hubbub/hubbub.go @@ -20,14 +20,14 @@ import ( "time" "github.com/google/go-github/v31/github" - "github.com/google/triage-party/pkg/initcache" + "github.com/google/triage-party/pkg/persist" ) // Config is how to configure a new hubbub engine type Config struct { - Client *github.Client // Client is a GitHub client - Cache initcache.Cacher // Cacher is a cache interface - Repos []string // Repos is the repositories to search + Client *github.Client // Client is a GitHub client + Cache persist.Cacher // Cacher is a cache interface + Repos []string // Repos is the repositories to search // Cache expiration times MemberRefresh time.Duration @@ -41,7 +41,7 @@ type Config struct { // Engine is the search engine interface for hubbub type Engine struct { - cache initcache.Cacher + cache persist.Cacher client *github.Client // How often to refresh organizational membership information diff --git a/pkg/hubbub/issue.go b/pkg/hubbub/issue.go index 541874b..a762f61 100644 --- a/pkg/hubbub/issue.go +++ b/pkg/hubbub/issue.go @@ -21,8 +21,8 @@ import ( "time" "github.com/google/go-github/v31/github" - "github.com/google/triage-party/pkg/initcache" "github.com/google/triage-party/pkg/logu" + "github.com/google/triage-party/pkg/persist" "gopkg.in/yaml.v2" "k8s.io/klog/v2" ) @@ -78,7 +78,7 @@ func (h *Engine) updateIssues(ctx context.Context, org string, project string, s opt.Page = resp.NextPage } - if err := h.cache.Set(key, &initcache.Hoard{Issues: allIssues}); err != nil { + if err := h.cache.Set(key, &persist.Thing{Issues: allIssues}); err != nil { klog.Errorf("set %q failed: %v", key, err) } @@ -121,7 +121,7 @@ func (h *Engine) updateIssueComments(ctx context.Context, org string, project st opt.Page = resp.NextPage } - if err := h.cache.Set(key, &initcache.Hoard{IssueComments: allComments}); err != nil { + if err := h.cache.Set(key, &persist.Thing{IssueComments: allComments}); err != nil { klog.Errorf("set %q failed: %v", key, err) } diff --git a/pkg/hubbub/match.go b/pkg/hubbub/match.go index 448aba2..0526b22 100644 --- a/pkg/hubbub/match.go +++ b/pkg/hubbub/match.go @@ -46,7 +46,7 @@ func preFetchMatch(i GitHubItem, labels []*github.Label, fs []Filter) bool { if f.Created != "" { if ok := matchDuration(i.GetCreatedAt(), f.Created); !ok { - klog.V(2).Infof("#%d creation at %s does not meet %s", i.GetNumber(), i.GetCreatedAt(), f.Created) + klog.V(2).Infof("#%d Created at %s does not meet %s", i.GetNumber(), i.GetCreatedAt(), f.Created) return false } } diff --git a/pkg/hubbub/orgs.go b/pkg/hubbub/orgs.go index e1fa6d4..ffa1a95 100644 --- a/pkg/hubbub/orgs.go +++ b/pkg/hubbub/orgs.go @@ -20,8 +20,8 @@ import ( "time" "github.com/google/go-github/v31/github" - "github.com/google/triage-party/pkg/initcache" "github.com/google/triage-party/pkg/logu" + "github.com/google/triage-party/pkg/persist" "k8s.io/klog/v2" ) @@ -53,7 +53,7 @@ func (h *Engine) cachedOrgMembers(ctx context.Context, org string, newerThan tim opt.Page = resp.NextPage } - if err := h.cache.Set(key, &initcache.Hoard{StringBool: members}); err != nil { + if err := h.cache.Set(key, &persist.Thing{StringBool: members}); err != nil { klog.Errorf("set %q failed: %v", key, err) } diff --git a/pkg/hubbub/pull_requests.go b/pkg/hubbub/pull_requests.go index 610d779..4d88d40 100644 --- a/pkg/hubbub/pull_requests.go +++ b/pkg/hubbub/pull_requests.go @@ -20,7 +20,7 @@ import ( "time" "github.com/google/go-github/v31/github" - "github.com/google/triage-party/pkg/initcache" + "github.com/google/triage-party/pkg/persist" "k8s.io/klog/v2" ) @@ -78,7 +78,7 @@ func (h *Engine) updatePRs(ctx context.Context, org string, project string, stat opt.Page = resp.NextPage } - if err := h.cache.Set(key, &initcache.Hoard{PullRequests: allPRs}); err != nil { + if err := h.cache.Set(key, &persist.Thing{PullRequests: allPRs}); err != nil { klog.Errorf("set %q failed: %v", key, err) } @@ -119,7 +119,7 @@ func (h *Engine) updatePRComments(ctx context.Context, org string, project strin opt.Page = resp.NextPage } - if err := h.cache.Set(key, &initcache.Hoard{PullRequestComments: allComments}); err != nil { + if err := h.cache.Set(key, &persist.Thing{PullRequestComments: allComments}); err != nil { klog.Errorf("set %q failed: %v", key, err) } diff --git a/pkg/initcache/mysql.go b/pkg/initcache/mysql.go deleted file mode 100644 index 57ee772..0000000 --- a/pkg/initcache/mysql.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package initcache provides a bootstrap for the in-memory cache - -package initcache - -import ( - "bufio" - "bytes" - "encoding/gob" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" - - "github.com/patrickmn/go-cache" - "k8s.io/klog" -) - -const ( - DiskExpireInterval = 65 * 24 * time.Hour - DiskCleanupInterval = 15 * time.Minute -) - -type MySQL struct { - path string - cache *cache.Cache - tableName -} - -// NewMySQL returns a new MySQL cache -func NewMySQL(cfg Config) *Disk { - gob.Register(&Hoard{}) - return &MySQL{path: cfg.Path, tableName: "initcache"} -} - -// Initialize creates or loads the disk cache -func (d *Disk) Initialize() error { - // Make cconnection - - klog.Infof("Initializing with %s ...", d.path) - if err := d.load(); err != nil { - klog.Infof("recreating cache due to load error: %v", err) - return d.create() - } - return nil -} - -func (d *Disk) load() error { - f, err := os.Open(d.path) - if err != nil { - return fmt.Errorf("open: %w", err) - } - defer f.Close() - - decoded := map[string]cache.Item{} - - gd := gob.NewDecoder(bufio.NewReader(f)) - - err = gd.Decode(&decoded) - if err != nil && err != io.EOF { - klog.Errorf("Decode failed: %v", err) - return d.create() - } - - if len(decoded) == 0 { - return fmt.Errorf("no items loaded from disk: %v", decoded) - } - - klog.Infof("%d items loaded from disk", len(decoded)) - d.cache = cache.NewFrom(DiskExpireInterval, DiskCleanupInterval, decoded) - return nil -} - -// Set stores a hoard onto disk -func (d *Disk) Set(key string, h *Hoard) error { - if h.Creation.IsZero() { - h.Creation = time.Now() - } - - d.cache.Set(key, h, DiskExpireInterval) - return nil -} - -// DeleteOlderThan deletes a hoard older than a timestamp -func (d *Disk) DeleteOlderThan(key string, t time.Time) error { - d.cache.Delete(key) - return nil -} - -// GetNewerThan returns a hoard older than a timestamp -func (d *Disk) GetNewerThan(key string, t time.Time) *Hoard { - x, ok := d.cache.Get(key) - if !ok { - klog.V(1).Infof("%s is not in the cache!", key) - return nil - } - - h := x.(*Hoard) - if h.Creation.Before(t) { - klog.V(1).Infof("%s is in cache, but %s is older than %s", key, h.Creation, t) - return nil - } - return h -} - -func (d *Disk) create() error { - klog.Infof("Creating cache, expire interval: %s", DiskExpireInterval) - - d.cache = cache.New(DiskExpireInterval, DiskCleanupInterval) - if err := d.Save(); err != nil { - return fmt.Errorf("save: %w", err) - } - return nil -} - -func (m *MySQL) Save() error { - start := time.Now() - items := d.cache.Items() - - klog.Infof("*** Saving %d items to initcache at %s", len(items), d.path) - defer func() { - klog.Infof("*** mysql.Save took %s", time.Since(start)) - }() - - for k, v := range items { - b := new(bytes.Buffer) - ge := gob.NewEncoder(b) - if err := ge.Encode(v); err != nil { - return fmt.Errorf("encode: %w", err) - } - - _, err := s.db.Exec(fmt.Sprintf(` - REPLACE INTO %s (key, value, ts) - VALUES (?, ?);`, m.tableName), key, b.Bytes(), start) - - if err != nil { - return fmt.Errorf("sql exec: %v (len=%d)", err, len(b)) - } - return nil - } - - // Flush older cache items out - var c string - err := s.db.Get(&c, fmt.Sprintf(`DELETE FROM %s WHERE ts > ?`, m.tableName), ts) - if err != nil { - return err - } -} \ No newline at end of file diff --git a/pkg/persist/cloudsql.go b/pkg/persist/cloudsql.go new file mode 100644 index 0000000..66dd14e --- /dev/null +++ b/pkg/persist/cloudsql.go @@ -0,0 +1,51 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package persist provides a persistence layer for the in-memory cache +package persist + +import ( + "strings" + + cloudsql "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/dialers/mysql" + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + "k8s.io/klog/v2" +) + +// NewCloudSQL returns a new Google Cloud SQL store (MySQL) +func NewCloudSQL(cfg *Config) (*MySQL, error) { + // DSN that works: + // $USER:$PASS@tcp($PROJECT/$REGION/$INSTANCE)/$DB" + dsn, err := mysql.ParseDSN(cfg.Path) + if err != nil { + return nil, err + } + + mcfg := cloudsql.Cfg(dsn.Addr, dsn.User, dsn.Passwd) + // Strip port + mcfg.Addr = strings.Split(dsn.Addr, ":")[0] + mcfg.Addr = strings.Replace(mcfg.Addr, "/", ":", -1) + mcfg.DBName = dsn.DBName + mcfg.ParseTime = true + klog.Infof("mcfg: %#v", mcfg) + + db, err := cloudsql.DialCfg(mcfg) + if err != nil { + return nil, err + } + + dbx := sqlx.NewDb(db, "mysql") + return &MySQL{db: dbx}, err +} diff --git a/pkg/initcache/disk.go b/pkg/persist/disk.go similarity index 57% rename from pkg/initcache/disk.go rename to pkg/persist/disk.go index bc8917f..8b44fc8 100644 --- a/pkg/initcache/disk.go +++ b/pkg/persist/disk.go @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package initcache provides a bootstrap for the in-memory cache +// Package persist provides a bootstrap for the in-memory cache -package initcache +package persist import ( "bufio" @@ -28,7 +28,6 @@ import ( "strings" "time" - "github.com/google/triage-party/pkg/logu" "github.com/patrickmn/go-cache" "k8s.io/klog/v2" ) @@ -45,16 +44,18 @@ type Disk struct { // NewDisk returns a new disk cache func NewDisk(cfg Config) *Disk { - gob.Register(&Hoard{}) + gob.Register(&Thing{}) return &Disk{path: cfg.Path} } -// Initialize creates or loads the disk cache func (d *Disk) Initialize() error { klog.Infof("Initializing with %s ...", d.path) if err := d.load(); err != nil { klog.Infof("recreating cache due to load error: %v", err) - return d.create() + d.cache = createMem() + if err := d.Save(); err != nil { + return fmt.Errorf("save: %w", err) + } } return nil } @@ -67,67 +68,37 @@ func (d *Disk) load() error { defer f.Close() decoded := map[string]cache.Item{} - gd := gob.NewDecoder(bufio.NewReader(f)) err = gd.Decode(&decoded) if err != nil && err != io.EOF { - klog.Errorf("Decode failed: %v", err) - return d.create() + return fmt.Errorf("decode failed: %w", err) } if len(decoded) == 0 { - return fmt.Errorf("no items loaded from disk: %v", decoded) + return fmt.Errorf("no items on disk") } klog.Infof("%d items loaded from disk", len(decoded)) - d.cache = cache.NewFrom(DiskExpireInterval, DiskCleanupInterval, decoded) + d.cache = loadMem(decoded) return nil } -// Set stores a hoard onto disk -func (d *Disk) Set(key string, h *Hoard) error { - if h.Creation.IsZero() { - h.Creation = time.Now() - } - - klog.V(1).Infof("Storing %s within in-memory cache", key) - d.cache.Set(key, h, DiskExpireInterval) +// Set stores a thing into memory +func (d *Disk) Set(key string, t *Thing) error { + setMem(d.cache, key, t) return nil } -// DeleteOlderThan deletes a hoard older than a timestamp +// DeleteOlderThan deletes a thing older than a timestamp func (d *Disk) DeleteOlderThan(key string, t time.Time) error { - d.cache.Delete(key) + deleteOlderMem(d.cache, key, t) return nil } -// GetNewerThan returns a hoard older than a timestamp -func (d *Disk) GetNewerThan(key string, t time.Time) *Hoard { - x, ok := d.cache.Get(key) - if !ok { - klog.Infof("%s is not within in-memory cache!", key) - return nil - } - - h := x.(*Hoard) - - if h.Creation.Before(t) { - klog.V(2).Infof("%s in cache, but %s is older than %s", key, logu.STime(h.Creation), logu.STime(t)) - return nil - } - - return h -} - -func (d *Disk) create() error { - klog.Infof("Creating in-memory cache, expire interval: %s", DiskExpireInterval) - - d.cache = cache.New(DiskExpireInterval, DiskCleanupInterval) - if err := d.Save(); err != nil { - return fmt.Errorf("save: %w", err) - } - return nil +// GetNewerThan returns a thing older than a timestamp +func (d *Disk) GetNewerThan(key string, t time.Time) *Thing { + return newerThanMem(d.cache, key, t) } func (d *Disk) Save() error { @@ -144,21 +115,46 @@ func (d *Disk) Save() error { if err := ge.Encode(items); err != nil { return fmt.Errorf("encode: %w", err) } + + if err := os.MkdirAll(filepath.Dir(d.path), 0700); err != nil { + return err + } + return ioutil.WriteFile(d.path, b.Bytes(), 0644) } -func DefaultDiskPath(configPath string, override string) string { - name := strings.Replace(filepath.Base(configPath), filepath.Ext(configPath), "", -1) +func findCacheRoot() string { + if _, err := os.Stat("/app/pcache"); err == nil { + return "/app/pcache" + } - if override != "" { - name = name + "_" + strings.Replace(override, "/", "_", -1) + if _, err := os.Stat("pcache"); err == nil { + return "pcache" + } + if _, err := os.Stat("../pcache"); err == nil { + return "../pcache" + } + if _, err := os.Stat("../../pcache"); err == nil { + return "../../pcache" } - // os.UserCacheDir() is technically better, but difficult to calculate in Dockerfile - home, err := os.UserHomeDir() + cdir, err := os.UserCacheDir() if err != nil { - klog.Exitf("unable to get home directory: %v", err) + return filepath.Join(os.TempDir(), "triage-party") + } + + return filepath.Join(cdir, "triage-party") +} + +func DefaultDiskPath(configPath string, override string) string { + + name := filepath.Base(configPath) + if override != "" { + name = name + "_" + strings.Replace(override, "/", "_", -1) } - return filepath.Join(home, ".tpcache", name) + dir := findCacheRoot() + path := filepath.Join(dir, name+".pc") + klog.Infof("default disk path: %s", path) + return path } diff --git a/pkg/persist/mem.go b/pkg/persist/mem.go new file mode 100644 index 0000000..167da09 --- /dev/null +++ b/pkg/persist/mem.go @@ -0,0 +1,76 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package persist provides a bootstrap for the in-memory cache + +package persist + +import ( + "time" + + "github.com/google/triage-party/pkg/logu" + "github.com/patrickmn/go-cache" + "k8s.io/klog" +) + +var ( + memExpireInterval = 65 * 24 * time.Hour + memCleanupInterval = 15 * time.Minute +) + +func createMem() *cache.Cache { + return cache.New(memExpireInterval, memCleanupInterval) +} + +func loadMem(items map[string]cache.Item) *cache.Cache { + return cache.NewFrom(memExpireInterval, memCleanupInterval, items) +} + +func setMem(c *cache.Cache, key string, th *Thing) { + if th.Created.IsZero() { + th.Created = time.Now() + } + + klog.V(1).Infof("Storing %s within in-memory cache", key) + c.Set(key, th, memExpireInterval) +} + +func newerThanMem(c *cache.Cache, key string, t time.Time) *Thing { + x, ok := c.Get(key) + if !ok { + klog.Infof("%s is not within in-memory cache!", key) + return nil + } + + th := x.(*Thing) + + if th.Created.Before(t) { + klog.V(2).Infof("%s in cache, but %s is older than %s", key, logu.STime(th.Created), logu.STime(t)) + return nil + } + + return th +} + +func deleteOlderMem(c *cache.Cache, key string, t time.Time) { + i := newerThanMem(c, key, t) + + // Still good. + if i != nil && i.Created.After(t) { + klog.Infof("no need to delete %s", key) + return + } + + c.Delete(key) +} diff --git a/pkg/persist/mysql.go b/pkg/persist/mysql.go new file mode 100644 index 0000000..4a6b262 --- /dev/null +++ b/pkg/persist/mysql.go @@ -0,0 +1,171 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package persist provides a persistence layer for the in-memory cache +package persist + +import ( + "bytes" + "encoding/gob" + "fmt" + "time" + + "github.com/jmoiron/sqlx" + "github.com/patrickmn/go-cache" + "k8s.io/klog" +) + +var schema = ` +CREATE TABLE IF NOT EXISTS persist ( + id INT AUTO_INCREMENT PRIMARY KEY, + key VARCHAR(255) NOT NULL, + ts TIMESTAMP DEFAULT '1970-01-01 00:00:01', + content MEDIUMBLOB, + + UNIQUE KEY unique_key (key), + INDEX ts_idx (ts), +); +` + +// sqlItem maps to schema +type sqlItem struct { + ID int64 `db:"id"` + Key string `db:"key"` + Created time.Time `db:"created"` + Saved time.Time `db:"saved"` + Value []byte `db:"content"` +} + +type MySQL struct { + cache *cache.Cache + db *sqlx.DB +} + +// NewMySQL returns a new MySQL cache +func NewMySQL(cfg Config) (*MySQL, error) { + dbx, err := sqlx.Connect("mysql", cfg.Path) + if err != nil { + return nil, err + } + + if _, err := dbx.Exec(schema); err != nil { + return nil, err + } + + m := &MySQL{ + db: dbx, + } + + if err := m.loadItems(); err != nil { + return m, fmt.Errorf("load: %w", err) + } + + return m, nil +} + +func (m *MySQL) loadItems() error { + rows, err := m.db.Queryx(`SELECT * FROM persist`) + if err != nil { + return fmt.Errorf("query: %w", err) + } + + decoded := map[string]cache.Item{} + + for rows.Next() { + var mi sqlItem + err = rows.StructScan(&mi) + if err != nil { + return fmt.Errorf("structscan: %w", err) + } + + var item cache.Item + gd := gob.NewDecoder(bytes.NewBuffer(mi.Value)) + if err := gd.Decode(&item); err != nil { + return fmt.Errorf("decode: %w", err) + } + decoded[mi.Key] = item + } + + if len(decoded) == 0 { + return fmt.Errorf("no items loaded from MySQL: %v", decoded) + } + + klog.Infof("%d items loaded from MySQL", len(decoded)) + m.cache = loadMem(decoded) + return nil +} + +// Set stores a thing +func (m *MySQL) Set(key string, th *Thing) error { + setMem(m.cache, key, th) + return nil +} + +// DeleteOlderThan deletes a thing older than a timestamp +func (m *MySQL) DeleteOlderThan(key string, t time.Time) error { + deleteOlderMem(m.cache, key, t) + return nil +} + +// GetNewerThan returns a Item older than a timestamp +func (m *MySQL) GetNewerThan(key string, t time.Time) *Thing { + return newerThanMem(m.cache, key, t) +} + +func (m *MySQL) Save() error { + start := time.Now() + items := m.cache.Items() + + klog.Infof("*** Saving %d items to MySQL", len(items)) + defer func() { + klog.Infof("*** mysql.Save took %s", time.Since(start)) + }() + + for k, v := range items { + b := new(bytes.Buffer) + ge := gob.NewEncoder(b) + if err := ge.Encode(v); err != nil { + return fmt.Errorf("encode: %w", err) + } + + // TODO: figure out how to get th.Created from v + x, ok := m.cache.Get(k) + if !ok { + klog.Errorf("expected %s to be in cache", k) + continue + } + th := x.(*Thing) + + _, err := m.db.Exec(` + INSERT INTO persist (key, value, ts) + VALUES (:key, :value, :created, :saved) + ON DUPLICATE KEY UPDATE + value = :value + created = :created + saved = :saved`, + sqlItem{Key: k, Value: b.Bytes(), Created: th.Created, Saved: start}) + + if err != nil { + return fmt.Errorf("sql exec: %v (len=%d)", err, len(b.Bytes())) + } + + return nil + } + + // Flush older cache items out + if _, err := m.db.Exec(`DELETE FROM persist WHERE saved < ?`, start); err != nil { + return err + } + return nil +} diff --git a/pkg/initcache/initcache.go b/pkg/persist/persist.go similarity index 75% rename from pkg/initcache/initcache.go rename to pkg/persist/persist.go index 28dd515..ed67a67 100644 --- a/pkg/initcache/initcache.go +++ b/pkg/persist/persist.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package initcache provides a bootstrap for the in-memory cache -package initcache +// Package persist provides a bootstrap for the in-memory cache +package persist import ( + "encoding/gob" "time" "github.com/google/go-github/v31/github" @@ -27,30 +28,29 @@ type Config struct { Path string } -type Hoard struct { - Creation time.Time - ID string - - PullRequests []*github.PullRequest - Issues []*github.Issue +type Thing struct { + Created time.Time + PullRequests []*github.PullRequest + Issues []*github.Issue PullRequestComments []*github.PullRequestComment IssueComments []*github.IssueComment - - StringBool map[string]bool + StringBool map[string]bool } // Cacher is the cache interface we support type Cacher interface { - Set(string, *Hoard) error + Set(string, *Thing) error DeleteOlderThan(string, time.Time) error - GetNewerThan(string, time.Time) *Hoard + GetNewerThan(string, time.Time) *Thing Initialize() error Save() error } func New(cfg Config) Cacher { + gob.Register(&Thing{}) + if cfg.Type == "disk" { return NewDisk(cfg) } diff --git a/pkg/triage/triage.go b/pkg/triage/triage.go index 7fbc816..f747b3f 100644 --- a/pkg/triage/triage.go +++ b/pkg/triage/triage.go @@ -22,14 +22,14 @@ import ( "github.com/google/go-github/v31/github" "github.com/google/triage-party/pkg/hubbub" - "github.com/google/triage-party/pkg/initcache" + "github.com/google/triage-party/pkg/persist" "gopkg.in/yaml.v2" "k8s.io/klog/v2" ) type Config struct { Client *github.Client - Cache initcache.Cacher + Cache persist.Cacher Repos []string MemberRefresh time.Duration // DebugNumber is useful when you want to debug why a single issue is or is-not appearing From b51ba5cb4150d7b93466b67b50b1f2aa9d064c4d Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 5 May 2020 09:03:15 -0700 Subject: [PATCH 3/6] MySQL persistence works! --- cmd/server/main.go | 33 +++++++------ cmd/tester/main.go | 16 +++---- examples/minikube-deploy.sh | 1 - examples/skaffold-deploy.sh | 1 - pkg/hubbub/similar.go | 2 - pkg/persist/cloudsql.go | 2 +- pkg/persist/disk.go | 9 ++-- pkg/persist/mem.go | 66 +++++++++++--------------- pkg/persist/mysql.go | 94 +++++++++++++++++++------------------ pkg/persist/persist.go | 46 ++++++++++++++++-- 10 files changed, 152 insertions(+), 118 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index c386ad2..57c797d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -12,6 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// It's the Triage Party server! +// +// ** Basic example: +// +// go run main.go --github-token-file ~/.token --config minikube.yaml +// +// ** Using MySQL persistence: +// +// --persist-backend=mysql --persist-path="root:rootz@tcp(127.0.0.1:3306)/teaparty" +// + package main import ( @@ -39,7 +50,7 @@ import ( var ( // shared with tester configPath = flag.String("config", "", "configuration path") - persistBackend = flag.String("persist-backend", "disk", "Cache persistence backend (disk, mysql, cloudsql)") + persistBackend = flag.String("persist-backend", "", "Cache persistence backend (disk, mysql, cloudsql)") persistPath = flag.String("persist-path", "", "Where to persist cache to (automatic)") reposOverride = flag.String("repos", "", "Override configured repos with this repository (comma separated)") @@ -76,20 +87,13 @@ func main() { klog.Exitf("open %s: %v", *configPath, err) } - cachePath := *persistPath - if *persistBackend == "disk" && cachePath == "" { - cachePath = persist.DefaultDiskPath(*configPath, *reposOverride) + c, err := persist.FromEnv(*persistBackend, *persistPath, *configPath, *reposOverride) + if err != nil { + klog.Exitf("unable to create persistence layer: %v", err) } - klog.Infof("cache path: %s", cachePath) - - c := persist.New(persist.Config{ - Type: *persistBackend, - Path: cachePath, - }) - if err := c.Initialize(); err != nil { - klog.Exitf("persist load to %s: %v", cachePath, err) + klog.Exitf("persist init with %s: %v", c, err) } cfg := triage.Config{ @@ -119,8 +123,9 @@ func main() { } // Make sure save works + klog.Infof("Validating persistence layer ...") if err := c.Save(); err != nil { - klog.Exitf("persist save to %s: %v", cachePath, err) + klog.Exitf("persist save to %s: %v", c, err) } u := updater.New(updater.Config{ @@ -147,7 +152,7 @@ func main() { for sig := range sigc { klog.Infof("signal caught: %v", sig) if err := c.Save(); err != nil { - klog.Errorf("save errro: %v", err) + klog.Errorf("unable to save: %v", err) } os.Exit(0) } diff --git a/cmd/tester/main.go b/cmd/tester/main.go index a2a8017..4a53f0f 100644 --- a/cmd/tester/main.go +++ b/cmd/tester/main.go @@ -32,9 +32,10 @@ import ( ) var ( - // shared with tester + // shared with server configPath = flag.String("config", "", "configuration path") - persistPath = flag.String("persist", "", "Where to load the initial cache from (optional)") + persistBackend = flag.String("persist-backend", "", "Cache persistence backend (disk, mysql, cloudsql)") + persistPath = flag.String("persist-path", "", "Where to persist cache to (automatic)") reposOverride = flag.String("repos", "", "Override configured repos with this repository (comma separated)") githubTokenFile = flag.String("github-token-file", "", "github token secret file, also settable via GITHUB_TOKEN") @@ -66,14 +67,13 @@ func main() { klog.Exitf("open %s: %v", *configPath, err) } - cachePath := *persistPath - if cachePath == "" { - cachePath = persist.DefaultDiskPath(*configPath, *reposOverride) + c, err := persist.FromEnv(*persistBackend, *persistPath, *configPath, *reposOverride) + if err != nil { + klog.Exitf("unable to create persistence layer: %v", err) } - c := persist.New(persist.Config{Type: "disk", Path: cachePath}) if err := c.Initialize(); err != nil { - klog.Exitf("persist load to %s: %v", cachePath, err) + klog.Exitf("persist initialize from %s: %v", c, err) } cfg := triage.Config{ @@ -99,7 +99,7 @@ func main() { } if err := c.Save(); err != nil { - klog.Exitf("persist save to %s: %v", cachePath, err) + klog.Exitf("persist save to %s: %v", c, err) } } diff --git a/examples/minikube-deploy.sh b/examples/minikube-deploy.sh index abd459e..70e35cd 100755 --- a/examples/minikube-deploy.sh +++ b/examples/minikube-deploy.sh @@ -34,6 +34,5 @@ gcloud beta run deploy "${SERVICE_NAME}" \ --set-env-vars="GITHUB_TOKEN=${token}" \ --allow-unauthenticated \ --region us-central1 \ - --max-instances 2 \ --memory 384Mi \ --platform managed diff --git a/examples/skaffold-deploy.sh b/examples/skaffold-deploy.sh index 11494dd..4cd2b5f 100755 --- a/examples/skaffold-deploy.sh +++ b/examples/skaffold-deploy.sh @@ -34,6 +34,5 @@ gcloud beta run deploy "${SERVICE_NAME}" \ --set-env-vars="GITHUB_TOKEN=${token}" \ --allow-unauthenticated \ --region us-central1 \ - --max-instances 2 \ --memory 384Mi \ --platform managed diff --git a/pkg/hubbub/similar.go b/pkg/hubbub/similar.go index cf14287..ae8b934 100644 --- a/pkg/hubbub/similar.go +++ b/pkg/hubbub/similar.go @@ -50,8 +50,6 @@ func (h *Engine) updateSimilarityTables(rawTitle, url string) { return } - klog.Infof("new title: %q", rawTitle) - // Update us -> them title similarity similarTo := []string{} diff --git a/pkg/persist/cloudsql.go b/pkg/persist/cloudsql.go index 66dd14e..0696c76 100644 --- a/pkg/persist/cloudsql.go +++ b/pkg/persist/cloudsql.go @@ -25,7 +25,7 @@ import ( ) // NewCloudSQL returns a new Google Cloud SQL store (MySQL) -func NewCloudSQL(cfg *Config) (*MySQL, error) { +func NewCloudSQL(cfg Config) (*MySQL, error) { // DSN that works: // $USER:$PASS@tcp($PROJECT/$REGION/$INSTANCE)/$DB" dsn, err := mysql.ParseDSN(cfg.Path) diff --git a/pkg/persist/disk.go b/pkg/persist/disk.go index 8b44fc8..46ead0d 100644 --- a/pkg/persist/disk.go +++ b/pkg/persist/disk.go @@ -43,9 +43,12 @@ type Disk struct { } // NewDisk returns a new disk cache -func NewDisk(cfg Config) *Disk { - gob.Register(&Thing{}) - return &Disk{path: cfg.Path} +func NewDisk(cfg Config) (*Disk, error) { + return &Disk{path: cfg.Path}, nil +} + +func (d *Disk) String() string { + return d.path } func (d *Disk) Initialize() error { diff --git a/pkg/persist/mem.go b/pkg/persist/mem.go index 167da09..7bd5bfd 100644 --- a/pkg/persist/mem.go +++ b/pkg/persist/mem.go @@ -19,58 +19,46 @@ package persist import ( "time" - "github.com/google/triage-party/pkg/logu" "github.com/patrickmn/go-cache" "k8s.io/klog" ) -var ( - memExpireInterval = 65 * 24 * time.Hour - memCleanupInterval = 15 * time.Minute -) - -func createMem() *cache.Cache { - return cache.New(memExpireInterval, memCleanupInterval) +type Memory struct { + cache *cache.Cache } -func loadMem(items map[string]cache.Item) *cache.Cache { - return cache.NewFrom(memExpireInterval, memCleanupInterval, items) +// NewMemory returns a new Memory cache +func NewMemory(cfg Config) (*Memory, error) { + return &Memory{}, nil } -func setMem(c *cache.Cache, key string, th *Thing) { - if th.Created.IsZero() { - th.Created = time.Now() - } - - klog.V(1).Infof("Storing %s within in-memory cache", key) - c.Set(key, th, memExpireInterval) +func (m *Memory) String() string { + return "memory" } -func newerThanMem(c *cache.Cache, key string, t time.Time) *Thing { - x, ok := c.Get(key) - if !ok { - klog.Infof("%s is not within in-memory cache!", key) - return nil - } - - th := x.(*Thing) - - if th.Created.Before(t) { - klog.V(2).Infof("%s in cache, but %s is older than %s", key, logu.STime(th.Created), logu.STime(t)) - return nil - } +func (m *Memory) Initialize() error { + m.cache = createMem() + return nil +} - return th +// Set stores a thing into memory +func (m *Memory) Set(key string, t *Thing) error { + setMem(m.cache, key, t) + return nil } -func deleteOlderMem(c *cache.Cache, key string, t time.Time) { - i := newerThanMem(c, key, t) +// DeleteOlderThan deletes a thing older than a timestamp +func (m *Memory) DeleteOlderThan(key string, t time.Time) error { + deleteOlderMem(m.cache, key, t) + return nil +} - // Still good. - if i != nil && i.Created.After(t) { - klog.Infof("no need to delete %s", key) - return - } +// GetNewerThan returns a thing older than a timestamp +func (m *Memory) GetNewerThan(key string, t time.Time) *Thing { + return newerThanMem(m.cache, key, t) +} - c.Delete(key) +func (m *Memory) Save() error { + klog.Warningf("Save is not implemented by the memory backend") + return nil } diff --git a/pkg/persist/mysql.go b/pkg/persist/mysql.go index 4a6b262..de3b6b0 100644 --- a/pkg/persist/mysql.go +++ b/pkg/persist/mysql.go @@ -29,52 +29,60 @@ import ( var schema = ` CREATE TABLE IF NOT EXISTS persist ( id INT AUTO_INCREMENT PRIMARY KEY, - key VARCHAR(255) NOT NULL, - ts TIMESTAMP DEFAULT '1970-01-01 00:00:01', - content MEDIUMBLOB, - - UNIQUE KEY unique_key (key), - INDEX ts_idx (ts), -); -` + saved TIMESTAMP DEFAULT '1970-01-01 00:00:01', + k VARCHAR(255) NOT NULL, + v MEDIUMBLOB, + UNIQUE KEY unique_k (k), + INDEX saved_idx (saved) +);` // sqlItem maps to schema type sqlItem struct { - ID int64 `db:"id"` - Key string `db:"key"` - Created time.Time `db:"created"` - Saved time.Time `db:"saved"` - Value []byte `db:"content"` + ID int64 `db:"id"` + Saved time.Time `db:"saved"` + Key string `db:"k"` + Value []byte `db:"v"` } type MySQL struct { cache *cache.Cache db *sqlx.DB + path string } // NewMySQL returns a new MySQL cache func NewMySQL(cfg Config) (*MySQL, error) { - dbx, err := sqlx.Connect("mysql", cfg.Path) + dbx, err := sqlx.Connect("mysql", cfg.Path+"?parseTime=true") if err != nil { return nil, err } - if _, err := dbx.Exec(schema); err != nil { - return nil, err + m := &MySQL{ + db: dbx, + path: cfg.Path, } - m := &MySQL{ - db: dbx, + return m, nil +} + +func (m *MySQL) String() string { + return fmt.Sprintf("mysql://%s", m.path) +} + +func (m *MySQL) Initialize() error { + if _, err := m.db.Exec(schema); err != nil { + return fmt.Errorf("exec schema: %w", err) } if err := m.loadItems(); err != nil { - return m, fmt.Errorf("load: %w", err) + return fmt.Errorf("load items: %w", err) } - return m, nil + return nil } func (m *MySQL) loadItems() error { + klog.Infof("loading items from persist table ...") rows, err := m.db.Queryx(`SELECT * FROM persist`) if err != nil { return fmt.Errorf("query: %w", err) @@ -97,10 +105,6 @@ func (m *MySQL) loadItems() error { decoded[mi.Key] = item } - if len(decoded) == 0 { - return fmt.Errorf("no items loaded from MySQL: %v", decoded) - } - klog.Infof("%d items loaded from MySQL", len(decoded)) m.cache = loadMem(decoded) return nil @@ -139,33 +143,33 @@ func (m *MySQL) Save() error { return fmt.Errorf("encode: %w", err) } - // TODO: figure out how to get th.Created from v - x, ok := m.cache.Get(k) - if !ok { - klog.Errorf("expected %s to be in cache", k) - continue + if _, err := m.db.Exec(` + INSERT INTO persist (k, v, saved) VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE k=VALUES(k), v=VALUES(v)`, + k, b.Bytes(), start); err != nil { + return fmt.Errorf("sql exec: %v (len=%d)", err, len(b.Bytes())) } - th := x.(*Thing) + } - _, err := m.db.Exec(` - INSERT INTO persist (key, value, ts) - VALUES (:key, :value, :created, :saved) - ON DUPLICATE KEY UPDATE - value = :value - created = :created - saved = :saved`, - sqlItem{Key: k, Value: b.Bytes(), Created: th.Created, Saved: start}) + return m.cleanup(start.Add(-1 * time.Hour)) +} - if err != nil { - return fmt.Errorf("sql exec: %v (len=%d)", err, len(b.Bytes())) - } +// Cleanup deletes older cache items +func (m *MySQL) cleanup(t time.Time) error { + res, err := m.db.Exec(`DELETE FROM persist WHERE saved < ?`, t) + + if err != nil { + return fmt.Errorf("delete exec: %w", err) + } - return nil + rows, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected: %w", err) } - // Flush older cache items out - if _, err := m.db.Exec(`DELETE FROM persist WHERE saved < ?`, start); err != nil { - return err + if rows > 0 { + klog.Infof("Deleted %d rows of stale data", rows) } + return nil } diff --git a/pkg/persist/persist.go b/pkg/persist/persist.go index ed67a67..974fca0 100644 --- a/pkg/persist/persist.go +++ b/pkg/persist/persist.go @@ -17,6 +17,8 @@ package persist import ( "encoding/gob" + "fmt" + "os" "time" "github.com/google/go-github/v31/github" @@ -40,6 +42,8 @@ type Thing struct { // Cacher is the cache interface we support type Cacher interface { + String() string + Set(string, *Thing) error DeleteOlderThan(string, time.Time) error GetNewerThan(string, time.Time) *Thing @@ -48,11 +52,45 @@ type Cacher interface { Save() error } -func New(cfg Config) Cacher { +func New(cfg Config) (Cacher, error) { gob.Register(&Thing{}) - - if cfg.Type == "disk" { + switch cfg.Type { + case "mysql": + return NewMySQL(cfg) + case "cloudsql": + return NewCloudSQL(cfg) + case "disk", "": return NewDisk(cfg) + case "memory": + return NewMemory(cfg) + default: + return nil, fmt.Errorf("unknown backend: %q", cfg.Type) + } +} + +// FromEnv is shared magic between binaries +func FromEnv(backend string, path string, configPath string, reposOverride string) (Cacher, error) { + if backend == "" { + backend = os.Getenv("PERSIST_BACKEND") + } + if backend == "" { + backend = "disk" + } + + if path == "" { + path = os.Getenv("PERSIST_PATH") + } + + if backend == "disk" && path == "" { + path = DefaultDiskPath(configPath, reposOverride) + } + + c, err := New(Config{ + Type: backend, + Path: path, + }) + if err != nil { + return nil, fmt.Errorf("new from %s: %s: %w", backend, path, err) } - return nil + return c, nil } From d5953283ca6ae4acbb232c4e8fd132683ac1b283 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 5 May 2020 10:28:24 -0700 Subject: [PATCH 4/6] Add docs, remove AcceptStaleResults hack --- README.md | 15 ++++++++- cmd/server/main.go | 6 ---- examples/minikube-deploy.sh | 2 +- pkg/hubbub/cache.go | 10 ------ pkg/hubbub/hubbub.go | 3 -- pkg/hubbub/search.go | 5 +-- pkg/hubbub/similar.go | 2 +- pkg/triage/rule.go | 2 +- pkg/triage/triage.go | 8 ----- pkg/updater/updater.go | 64 ++++++++++++++++++++++--------------- 10 files changed, 58 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 84c508a..f43a5e4 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,6 @@ collections: The first rule, `discuss`, include all items labelled as `triage/discuss`, whether they are pull requests or issues, open or closed. - ```yaml rules: discuss: @@ -172,3 +171,17 @@ See [examples/minikube-deploy.sh](examples/minikube-deploy.sh) Kubernetes: See [examples/generic-kubernetes.yaml](examples/generic-kubernetes.yaml) + +## Configuring Persistence + +Triage Party uses an in-memory cache with an optional persistence layer to decrease the load on GitHub API. By default, Triage Party persists occasionally to disk, but it is configurable via: + +* Type: `--persist-backend` flag or `PERSIST_BACKEND` environment variable +* Path: `--persist-path` flag or `PERSIST_PATH` environment flag. + +Examples: + +* **Custom disk path**: `--persist-path=/var/tmp/tp` +* **MySQL**: `--persist-backend=mysql --persist-path="user:password@tcp(127.0.0.1:3306)/tp"` +* **CloudSQL (MySQL)**: `--persist-backend=cloudsql --persist-path="user:password@tcp(project/us-central1/triage-party)/db"` + * May require configuring [GOOGLE_APPLICATION_CREDENTIALS](https://cloud.google.com/docs/authentication/getting-started) diff --git a/cmd/server/main.go b/cmd/server/main.go index 57c797d..2ac7ca5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -122,12 +122,6 @@ func main() { sn = calculateSiteName(ts) } - // Make sure save works - klog.Infof("Validating persistence layer ...") - if err := c.Save(); err != nil { - klog.Exitf("persist save to %s: %v", c, err) - } - u := updater.New(updater.Config{ Party: tp, MinRefresh: *minRefresh, diff --git a/examples/minikube-deploy.sh b/examples/minikube-deploy.sh index 70e35cd..625f7e8 100755 --- a/examples/minikube-deploy.sh +++ b/examples/minikube-deploy.sh @@ -31,7 +31,7 @@ readonly token="$(cat ${GITHUB_TOKEN_PATH})" gcloud beta run deploy "${SERVICE_NAME}" \ --project "${PROJECT}" \ --image "${IMAGE}" \ - --set-env-vars="GITHUB_TOKEN=${token}" \ + --set-env-vars="GITHUB_TOKEN=${token},PERSIST_BACKEND=cloudsql,PERSIST_PATH=tp:${DB_PASS}@tcp(k8s-minikube/us-central1/triage-party)/tp" \ --allow-unauthenticated \ --region us-central1 \ --memory 384Mi \ diff --git a/pkg/hubbub/cache.go b/pkg/hubbub/cache.go index c08d459..c29e132 100644 --- a/pkg/hubbub/cache.go +++ b/pkg/hubbub/cache.go @@ -7,18 +7,8 @@ import ( "k8s.io/klog/v2" ) -// Toggle acceptability of stale results, useful for bootstrapping -func (e *Engine) AcceptStaleResults(b bool) { - klog.V(1).Infof("Setting stale results=%v", b) - e.acceptStaleResults = b -} - // FlushSearchCache invalidates the in-memory search cache func (h *Engine) FlushSearchCache(org string, project string, olderThan time.Time) error { - if h.acceptStaleResults { - return fmt.Errorf("stale results enabled, refusing to flush") - } - h.flushIssueSearchCache(org, project, olderThan) h.flushPRSearchCache(org, project, olderThan) return nil diff --git a/pkg/hubbub/hubbub.go b/pkg/hubbub/hubbub.go index 7deb585..9d994d3 100644 --- a/pkg/hubbub/hubbub.go +++ b/pkg/hubbub/hubbub.go @@ -57,9 +57,6 @@ type Engine struct { // indexes used for similarity matching seen map[string]*Conversation - - // are stale results acceptable? - acceptStaleResults bool } func New(cfg Config) *Engine { diff --git a/pkg/hubbub/search.go b/pkg/hubbub/search.go index 82932a2..a3aaf7e 100644 --- a/pkg/hubbub/search.go +++ b/pkg/hubbub/search.go @@ -41,8 +41,9 @@ func (h *Engine) SearchIssues(ctx context.Context, org string, project string, f var err error orgCutoff := time.Now().Add(h.memberRefresh * -1) - if h.acceptStaleResults { - orgCutoff = time.Time{} + if orgCutoff.After(newerThan) { + klog.Infof("Setting org cutoff to %s", newerThan) + orgCutoff = newerThan } wg.Add(1) diff --git a/pkg/hubbub/similar.go b/pkg/hubbub/similar.go index ae8b934..18e81f8 100644 --- a/pkg/hubbub/similar.go +++ b/pkg/hubbub/similar.go @@ -60,7 +60,7 @@ func (h *Engine) updateSimilarityTables(rawTitle, url string) { } if godice.CompareString(title, otherTitle) > h.MinSimilarity { - klog.Infof("%q is similar to %q", rawTitle, otherTitle) + klog.V(1).Infof("%q is similar to %q", rawTitle, otherTitle) similarTo = append(similarTo, otherTitle) } return true diff --git a/pkg/triage/rule.go b/pkg/triage/rule.go index 27efd8a..130329b 100644 --- a/pkg/triage/rule.go +++ b/pkg/triage/rule.go @@ -103,7 +103,7 @@ func SummarizeRuleResult(t Rule, cs []*hubbub.Conversation, seen map[string]*Rul // ExecuteRule executes a rule. seen is optional. func (p *Party) ExecuteRule(ctx context.Context, t Rule, seen map[string]*Rule, newerThan time.Time) (*RuleResult, error) { - klog.Infof("executing rule %q for results newer than %s (stale_ok=%v)", t.ID, logu.STime(newerThan), p.acceptStaleResults) + klog.Infof("executing rule %q for results newer than %s", t.ID, logu.STime(newerThan)) rcs := []*hubbub.Conversation{} for _, repo := range t.Repos { diff --git a/pkg/triage/triage.go b/pkg/triage/triage.go index f747b3f..77e6a87 100644 --- a/pkg/triage/triage.go +++ b/pkg/triage/triage.go @@ -43,8 +43,6 @@ type Party struct { rules map[string]Rule reposOverride []string debugNumber int - - acceptStaleResults bool } func New(cfg Config) *Party { @@ -224,9 +222,3 @@ func processRules(raw map[string]Rule) (map[string]Rule, error) { return rules, nil } - -// Toggle acceptability of stale results, useful for bootstrapping -func (p *Party) AcceptStaleResults(b bool) { - p.acceptStaleResults = b - p.engine.AcceptStaleResults(b) -} diff --git a/pkg/updater/updater.go b/pkg/updater/updater.go index 71b3537..b6acbfc 100644 --- a/pkg/updater/updater.go +++ b/pkg/updater/updater.go @@ -18,6 +18,7 @@ package updater import ( "context" "fmt" + "math/rand" "sync" "time" @@ -63,7 +64,8 @@ type Updater struct { cache map[string]*triage.CollectionResult lastRequest sync.Map secondLastRequest sync.Map - lastSave time.Time + lastPersist time.Time + lastRun time.Time startTime time.Time loopEvery time.Duration mutex *sync.Mutex @@ -189,15 +191,14 @@ func (u *Updater) secondLastRequested(id string) time.Time { } func (u *Updater) update(ctx context.Context, s triage.Collection) error { - if u.lastSave.IsZero() { + cutoff := time.Now().Add(minFlushAge * -1) + if u.lastPersist.IsZero() { klog.Infof("have not yet saved content - will accept stale results") - u.party.AcceptStaleResults(true) - } else { - u.party.AcceptStaleResults(false) + cutoff = time.Time{} } klog.Infof(">>> updating %q >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>", s.ID) - r, err := u.party.ExecuteCollection(ctx, s, time.Now()) + r, err := u.party.ExecuteCollection(ctx, s, cutoff) if err != nil { return err } @@ -230,8 +231,23 @@ func (u *Updater) RunSingle(ctx context.Context, id string, force bool) (bool, e return updated, nil } +// Persist saves results to the persistence layer +func (u *Updater) Persist() error { + u.lastPersist = time.Now() + + if err := u.persistFunc(); err != nil { + return err + } + + return nil +} + // Run once, optionally forcing an update func (u *Updater) RunOnce(ctx context.Context, force bool) error { + defer func() { + u.lastRun = time.Now() + }() + updated := false if force { klog.Warningf(">>> RunOnce has force enabled") @@ -243,6 +259,11 @@ func (u *Updater) RunOnce(ctx context.Context, force bool) error { return err } + if u.lastRun.IsZero() { + u.startTime = time.Now() + force = true + } + var failed []string for _, s := range sts { runUpdated, err := u.RunSingle(ctx, s.ID, force) @@ -255,12 +276,16 @@ func (u *Updater) RunOnce(ctx context.Context, force bool) error { } } - if updated && time.Since(u.lastSave) > u.maxRefresh { - if err := u.persistFunc(); err != nil { - klog.Errorf("persist failed: %v", err) - } else { - u.lastSave = time.Now() - } + cutoff := u.maxRefresh + time.Duration(rand.Intn(int(u.maxRefresh.Seconds())))*time.Second + sinceSave := time.Since(u.lastPersist) + + if updated && sinceSave > cutoff { + klog.Infof("%s since cache has been saved (cutoff=%s)", cutoff, sinceSave) + go func() { + if err := u.Persist(); err != nil { + klog.Errorf("persist failed: %v", err) + } + }() } if len(failed) > 0 { @@ -272,21 +297,8 @@ func (u *Updater) RunOnce(ctx context.Context, force bool) error { // Update loop func (u *Updater) Loop(ctx context.Context) error { - klog.Infof("Looping: data will be updated between %s and %s", u.minRefresh, u.maxRefresh) - - klog.Infof("Generating results from stale data ...") - if err := u.RunOnce(ctx, false); err != nil { - return err - } - - klog.Infof("Generating results from fresh data ...") - u.startTime = time.Now() - if err := u.RunOnce(ctx, true); err != nil { - return err - } - // Loop if everything goes to plan - klog.Infof("Results are now fresh, starting refresh loop ...") + klog.Infof("Looping: data will be updated between %s and %s", u.minRefresh, u.maxRefresh) ticker := time.NewTicker(u.loopEvery) defer ticker.Stop() for range ticker.C { From 869ff07942124698012c167509c67fcd1988070e Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 5 May 2020 13:02:52 -0700 Subject: [PATCH 5/6] Make stale data loads show a reasonable result age --- cmd/server/main.go | 2 +- pkg/hubbub/search.go | 34 ++++++++++++++++++++++++---------- pkg/site/site.go | 31 +++++++++++++++++-------------- pkg/triage/collection.go | 16 +++++++++++++--- pkg/triage/rule.go | 19 ++++++++++++++----- pkg/triage/triage.go | 2 ++ site/collection.tmpl | 2 +- 7 files changed, 72 insertions(+), 34 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 2ac7ca5..72a7806 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -162,7 +162,7 @@ func main() { BaseDirectory: findPath(*siteDir), Updater: u, Party: tp, - WarnAge: 2 * *maxRefresh, + WarnAge: *maxRefresh + 5*(time.Minute), Name: sn, }) diff --git a/pkg/hubbub/search.go b/pkg/hubbub/search.go index a3aaf7e..df2b76b 100644 --- a/pkg/hubbub/search.go +++ b/pkg/hubbub/search.go @@ -15,22 +15,26 @@ import ( ) // Search for GitHub issues or PR's -func (h *Engine) SearchAny(ctx context.Context, org string, project string, fs []Filter, newerThan time.Time) ([]*Conversation, error) { - cs, err := h.SearchIssues(ctx, org, project, fs, newerThan) +func (h *Engine) SearchAny(ctx context.Context, org string, project string, fs []Filter, newerThan time.Time) ([]*Conversation, time.Time, error) { + cs, ts, err := h.SearchIssues(ctx, org, project, fs, newerThan) if err != nil { - return cs, err + return cs, ts, err } - pcs, err := h.SearchPullRequests(ctx, org, project, fs, newerThan) + pcs, pts, err := h.SearchPullRequests(ctx, org, project, fs, newerThan) if err != nil { - return cs, err + return cs, ts, err } - return append(cs, pcs...), nil + if pts.After(ts) { + ts = pts + } + + return append(cs, pcs...), ts, nil } // Search for GitHub issues or PR's -func (h *Engine) SearchIssues(ctx context.Context, org string, project string, fs []Filter, newerThan time.Time) ([]*Conversation, error) { +func (h *Engine) SearchIssues(ctx context.Context, org string, project string, fs []Filter, newerThan time.Time) ([]*Conversation, time.Time, error) { fs = openByDefault(fs) klog.V(1).Infof("Gathering raw data for %s/%s search %s - newer than %s", org, project, toYAML(fs), logu.STime(newerThan)) var wg sync.WaitGroup @@ -80,9 +84,14 @@ func (h *Engine) SearchIssues(ctx context.Context, org string, project string, f wg.Wait() var is []*github.Issue + var latest time.Time seen := map[string]bool{} for _, i := range append(open, closed...) { + if i.GetUpdatedAt().After(latest) { + latest = i.GetUpdatedAt() + } + if h.debugNumber != 0 { if i.GetNumber() == h.debugNumber { klog.Errorf("*** Found debug issue #%d:\n%s", i.GetNumber(), formatStruct(*i)) @@ -140,10 +149,10 @@ func (h *Engine) SearchIssues(ctx context.Context, org string, project string, f } klog.V(1).Infof("%d of %d issues within %s/%s matched filters %s", len(filtered), len(is), org, project, toYAML(fs)) - return filtered, nil + return filtered, latest, nil } -func (h *Engine) SearchPullRequests(ctx context.Context, org string, project string, fs []Filter, newerThan time.Time) ([]*Conversation, error) { +func (h *Engine) SearchPullRequests(ctx context.Context, org string, project string, fs []Filter, newerThan time.Time) ([]*Conversation, time.Time, error) { fs = openByDefault(fs) klog.V(1).Infof("Searching %s/%s for PR's matching: %s - newer than %s", org, project, toYAML(fs), logu.STime(newerThan)) @@ -179,8 +188,13 @@ func (h *Engine) SearchPullRequests(ctx context.Context, org string, project str wg.Wait() + var latest time.Time prs := []*github.PullRequest{} for _, pr := range append(open, closed...) { + if pr.GetUpdatedAt().After(latest) { + latest = pr.GetUpdatedAt() + } + if h.debugNumber != 0 { if pr.GetNumber() == h.debugNumber { klog.Errorf("*** Found debug PR #%d:\n%s", pr.GetNumber(), formatStruct(*pr)) @@ -225,7 +239,7 @@ func (h *Engine) SearchPullRequests(ctx context.Context, org string, project str } klog.V(1).Infof("%d of %d PR's within %s/%s matched filters:\n%s", len(filtered), len(prs), org, project, toYAML(fs)) - return filtered, nil + return filtered, latest, nil } func formatStruct(x interface{}) string { diff --git a/pkg/site/site.go b/pkg/site/site.go index 4199a65..6177e47 100644 --- a/pkg/site/site.go +++ b/pkg/site/site.go @@ -55,21 +55,23 @@ type Config struct { func New(c *Config) *Handlers { return &Handlers{ - baseDir: c.BaseDirectory, - updater: c.Updater, - party: c.Party, - siteName: c.Name, - warnAge: c.WarnAge, + baseDir: c.BaseDirectory, + updater: c.Updater, + party: c.Party, + siteName: c.Name, + warnAge: c.WarnAge, + startTime: time.Now(), } } // Handlers is a mix of config and client interfaces to connect with. type Handlers struct { - baseDir string - updater *updater.Updater - party *triage.Party - siteName string - warnAge time.Duration + baseDir string + updater *updater.Updater + party *triage.Party + siteName string + warnAge time.Duration + startTime time.Time } // Root redirects to leaderboard. @@ -202,8 +204,10 @@ func (h *Handlers) Collection() http.HandlerFunc { } warning := "" - if time.Since(result.Time) > h.warnAge { - warning = fmt.Sprintf("Serving stale results (%s old) - refreshing results in background. Use Shift-Reload to force data to refresh at any time.", time.Since(result.Time)) + if result.Time.IsZero() { + warning = fmt.Sprintf("Triage Party started %s ago, and is serving stale results while refreshing in the background", humanDuration(time.Since(h.startTime))) + } else if time.Since(result.Time) > h.warnAge { + warning = fmt.Sprintf("Serving stale results (%s old) - refreshing results in background. Use Shift-Reload to force data to refresh at any time.", humanDuration(time.Since(result.Time))) } total := 0 @@ -376,7 +380,6 @@ func avatar(u *github.User) template.HTML { func playerFilter(result *triage.CollectionResult, player int, players int) *triage.CollectionResult { klog.Infof("Filtering for player %d of %d ...", player, players) os := []*triage.RuleResult{} - seen := map[string]*triage.Rule{} for _, o := range result.RuleResults { @@ -390,7 +393,7 @@ func playerFilter(result *triage.CollectionResult, player int, players int) *tri } os = append(os, triage.SummarizeRuleResult(o.Rule, cs, seen)) - } + return triage.SummarizeCollectionResult(os) } diff --git a/pkg/triage/collection.go b/pkg/triage/collection.go index 0a43bd7..3de7809 100644 --- a/pkg/triage/collection.go +++ b/pkg/triage/collection.go @@ -36,7 +36,8 @@ type Collection struct { // The result of Execute type CollectionResult struct { - Time time.Time + Time time.Time + RuleResults []*RuleResult Total int @@ -60,6 +61,7 @@ func (p *Party) ExecuteCollection(ctx context.Context, s Collection, newerThan t os := []*RuleResult{} seen := map[string]*Rule{} seenRule := map[string]bool{} + var latestInput time.Time for _, tid := range s.RuleIDs { if seenRule[tid] { @@ -79,13 +81,21 @@ func (p *Party) ExecuteCollection(ctx context.Context, s Collection, newerThan t return nil, fmt.Errorf("rule %q: %v", t.Name, err) } + if ro.LatestInput.After(latestInput) { + latestInput = ro.LatestInput + } + os = append(os, ro) } r := SummarizeCollectionResult(os) - // More accurate than time.Now() - r.Time = newerThan + // If we're starting from a stale cache, use the most recent underlying input + if newerThan.IsZero() { + r.Time = latestInput + } else { + r.Time = newerThan + } klog.Infof("<<< Collection %q took %s to execute", s.ID, time.Since(start)) return r, nil diff --git a/pkg/triage/rule.go b/pkg/triage/rule.go index 130329b..e8a65bb 100644 --- a/pkg/triage/rule.go +++ b/pkg/triage/rule.go @@ -48,6 +48,9 @@ type RuleResult struct { TotalAccumulatedHoldDays float64 Duplicates int + + // LatestInput is the timestamp of the most recent input data + LatestInput time.Time } // SummarizeRuleResult adds together statistics about a pool of conversations @@ -105,6 +108,7 @@ func SummarizeRuleResult(t Rule, cs []*hubbub.Conversation, seen map[string]*Rul func (p *Party) ExecuteRule(ctx context.Context, t Rule, seen map[string]*Rule, newerThan time.Time) (*RuleResult, error) { klog.Infof("executing rule %q for results newer than %s", t.ID, logu.STime(newerThan)) rcs := []*hubbub.Conversation{} + var latest time.Time for _, repo := range t.Repos { org, project, err := parseRepo(repo) @@ -114,15 +118,15 @@ func (p *Party) ExecuteRule(ctx context.Context, t Rule, seen map[string]*Rule, klog.V(2).Infof("%s -> org=%s project=%s", repo, org, project) + var ts time.Time var cs []*hubbub.Conversation switch t.Type { case hubbub.Issue: - - cs, err = p.engine.SearchIssues(ctx, org, project, t.Filters, newerThan) + cs, ts, err = p.engine.SearchIssues(ctx, org, project, t.Filters, newerThan) case hubbub.PullRequest: - cs, err = p.engine.SearchPullRequests(ctx, org, project, t.Filters, newerThan) + cs, ts, err = p.engine.SearchPullRequests(ctx, org, project, t.Filters, newerThan) default: - cs, err = p.engine.SearchAny(ctx, org, project, t.Filters, newerThan) + cs, ts, err = p.engine.SearchAny(ctx, org, project, t.Filters, newerThan) } if err != nil { @@ -130,10 +134,15 @@ func (p *Party) ExecuteRule(ctx context.Context, t Rule, seen map[string]*Rule, } rcs = append(rcs, cs...) + if ts.After(latest) { + latest = ts + } } klog.V(1).Infof("rule %q matched %d items", t.ID, len(rcs)) - return SummarizeRuleResult(t, rcs, seen), nil + rr := SummarizeRuleResult(t, rcs, seen) + rr.LatestInput = latest + return rr, nil } // Return a fully resolved rule diff --git a/pkg/triage/triage.go b/pkg/triage/triage.go index 77e6a87..d21b579 100644 --- a/pkg/triage/triage.go +++ b/pkg/triage/triage.go @@ -40,6 +40,7 @@ type Party struct { engine *hubbub.Engine settings Settings collections []Collection + cache persist.Cacher rules map[string]Rule reposOverride []string debugNumber int @@ -59,6 +60,7 @@ func New(cfg Config) *Party { return &Party{ engine: h, + cache: cfg.Cache, reposOverride: cfg.Repos, debugNumber: cfg.DebugNumber, } diff --git a/site/collection.tmpl b/site/collection.tmpl index c19ef66..47d3eee 100644 --- a/site/collection.tmpl +++ b/site/collection.tmpl @@ -15,7 +15,7 @@
- {{ if eq .TotalShown .Total }}{{ .Total }} unique items{{ else }}Showing {{ .TotalShown }} of {{ .Total}} unique items{{ end }}, + {{ if eq .TotalShown .Total }}{{ .Total }} unique items{{ else }}Showing {{ .TotalShown }} of {{ .Total}} unique items{{ end }}, Avg age: {{ .CollectionResult.AvgAge | toDays }}, Avg wait: {{ .CollectionResult.AvgCurrentHold | toDays }} From fba6f8fec8eb77b08d17bfdc0556c62602178bca Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 5 May 2020 13:12:41 -0700 Subject: [PATCH 6/6] Add missing files --- .gitignore | 22 ++++++++++++ pcache/README.md | 1 + pkg/logu/logu.go | 28 ++++++++++++++++ pkg/persist/gocache.go | 76 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 .gitignore create mode 100644 pcache/README.md create mode 100644 pkg/logu/logu.go create mode 100644 pkg/persist/gocache.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f1a00a --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +vendor/ + +# persist cache output +*.pc + +# binaries +main diff --git a/pcache/README.md b/pcache/README.md new file mode 100644 index 0000000..8b272d4 --- /dev/null +++ b/pcache/README.md @@ -0,0 +1 @@ +persistent cache files diff --git a/pkg/logu/logu.go b/pkg/logu/logu.go new file mode 100644 index 0000000..3c04a20 --- /dev/null +++ b/pkg/logu/logu.go @@ -0,0 +1,28 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// logu are helpers that are just good for logging +package logu + +import ( + "fmt" + "time" +) + +// STime returns a log-friendly timestamp +func STime(t time.Time) string { + _, month, day := t.Date() + hour, minute, second := t.Clock() + return fmt.Sprintf("%02d%02d %02d:%02d:%02d", int(month), day, hour, minute, second) +} diff --git a/pkg/persist/gocache.go b/pkg/persist/gocache.go new file mode 100644 index 0000000..4710e24 --- /dev/null +++ b/pkg/persist/gocache.go @@ -0,0 +1,76 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package persist provides a bootstrap for the in-memory cache + +package persist + +import ( + "time" + + "github.com/google/triage-party/pkg/logu" + "github.com/patrickmn/go-cache" + "k8s.io/klog" +) + +var ( + memExpireInterval = 65 * 24 * time.Hour + memCleanupInterval = 15 * time.Minute +) + +func createMem() *cache.Cache { + return cache.New(memExpireInterval, memCleanupInterval) +} + +func loadMem(items map[string]cache.Item) *cache.Cache { + return cache.NewFrom(memExpireInterval, memCleanupInterval, items) +} + +func setMem(c *cache.Cache, key string, th *Thing) { + if th.Created.IsZero() { + th.Created = time.Now() + } + + klog.V(1).Infof("Storing %s within in-memory cache", key) + c.Set(key, th, memExpireInterval) +} + +func newerThanMem(c *cache.Cache, key string, t time.Time) *Thing { + x, ok := c.Get(key) + if !ok { + klog.V(1).Infof("%s is not within in-memory cache!", key) + return nil + } + + th := x.(*Thing) + + if th.Created.Before(t) { + klog.V(2).Infof("%s in cache, but %s is older than %s", key, logu.STime(th.Created), logu.STime(t)) + return nil + } + + return th +} + +func deleteOlderMem(c *cache.Cache, key string, t time.Time) { + i := newerThanMem(c, key, t) + + // Still good. + if i != nil && i.Created.After(t) { + klog.Infof("no need to delete %s", key) + return + } + + c.Delete(key) +}