Merge pull request #55 from tstromberg/mysql

Persistent cache refactor with MySQL support

.gitignore

@@ -0,0 +1,22 @@
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+vendor/
+
+# persist cache output
+*.pc
+
+# binaries
+main

Dockerfile

@@ -13,33 +13,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang
+FROM golang AS builder
 WORKDIR /app
 
-# CFG is the path to your configuration file
+# CFG is the path to your Triage Party configuration
 ARG CFG
 
-# Set an env var that matches your github repo name, replace treeder/dockergo here with your repo name
+# Build the binary
 ENV SRC_DIR=/src/tparty
 ENV GO111MODULE=on
-
 RUN mkdir -p ${SRC_DIR}/cmd ${SRC_DIR}/third_party ${SRC_DIR}/pkg ${SRC_DIR}/site /app/third_party /app/site
 COPY go.* $SRC_DIR/
 COPY cmd ${SRC_DIR}/cmd/
 COPY pkg ${SRC_DIR}/pkg/
+WORKDIR $SRC_DIR
+RUN go mod download
+RUN go build cmd/server/main.go
 
-# Build the binary
-RUN cd $SRC_DIR && go mod download
-RUN cd $SRC_DIR/cmd/server && go build -o main
-RUN cp $SRC_DIR/cmd/server/main /app/
+# Populate disk cache data (optional)
+FROM alpine AS persist
+ARG CFG
+COPY pcache /pc
+RUN echo "failure is OK with this next step (cache population)"
+RUN mv /pc/$(basename $CFG).pc /config.yaml.pc || touch /config.yaml.pc
 
-# Setup our deployment
+# Setup the site data
+FROM gcr.io/distroless/base
+ARG CFG
+COPY --from=builder /src/tparty/main /app/
+COPY --from=persist /config.yaml.pc /app/pcache/config.yaml.pc
 COPY site /app/site/
 COPY third_party /app/third_party/
 COPY $CFG /app/config.yaml
 
-# Bad hack: pre-heat the cache in lieu of persistent storage
-RUN --mount=type=secret,id=github /app/main --github-token-file=/run/secrets/github --config /app/config.yaml --site /app/site --dry-run
-
 # Run the server at a reasonable refresh rate
 CMD ["/app/main", "--min-refresh=25s", "--max-refresh=20m", "--config=/app/config.yaml", "--site=/app/site", "--3p=/app/third_party"]

README.md

@@ -77,7 +77,6 @@ collections:
 
 The first rule, `discuss`, include all items labelled as `triage/discuss`, whether they are pull requests or issues, open or closed.
 
-
 ```yaml
 rules:
   discuss:
@@ -161,13 +160,8 @@ For full example configurations, see `examples/*.yaml`. There are two that are p
 Docker:
 
 ```shell
-env DOCKER_BUILDKIT=1 \
-  GITHUB_TOKEN_PATH=<path to your github token> \
-  docker build --tag=tp \
-  --build-arg CFG=examples/generic-project.yaml \
-  --secret id=github,src=$GITHUB_TOKEN_PATH .
-
-docker run -p 8080:8080 tp
+docker build --tag=tp --build-arg CFG=examples/generic-project.yaml .
+docker run -e GITHUB_TOKEN=<your token> -p 8080:8080 tp
 ```
 
 Cloud Run:
@@ -177,3 +171,17 @@ See [examples/minikube-deploy.sh](examples/minikube-deploy.sh)
 Kubernetes:
 
 See [examples/generic-kubernetes.yaml](examples/generic-kubernetes.yaml)
+
+## Configuring Persistence
+
+Triage Party uses an in-memory cache with an optional persistence layer to decrease the load on GitHub API. By default, Triage Party persists occasionally to disk, but it is configurable via:
+
+* Type: `--persist-backend` flag or `PERSIST_BACKEND` environment variable
+* Path: `--persist-path` flag or `PERSIST_PATH` environment flag.
+
+Examples:
+
+* **Custom disk path**: `--persist-path=/var/tmp/tp`
+* **MySQL**: `--persist-backend=mysql --persist-path="user:password@tcp(127.0.0.1:3306)/tp"`
+* **CloudSQL (MySQL)**: `--persist-backend=cloudsql --persist-path="user:password@tcp(project/us-central1/triage-party)/db"`
+  * May require configuring [GOOGLE_APPLICATION_CREDENTIALS](https://cloud.google.com/docs/authentication/getting-started)

cmd/server/main.go

@@ -12,6 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// It's the Triage Party server!
+//
+// ** Basic example:
+//
+// go run main.go --github-token-file ~/.token --config minikube.yaml
+//
+// ** Using MySQL persistence:
+//
+// --persist-backend=mysql --persist-path="root:rootz@tcp(127.0.0.1:3306)/teaparty"
+//
+
 package main
 
 import (
@@ -30,16 +41,18 @@ import (
 	"golang.org/x/oauth2"
 	"k8s.io/klog/v2"
 
-	"github.com/google/triage-party/pkg/initcache"
+	"github.com/google/triage-party/pkg/persist"
 	"github.com/google/triage-party/pkg/site"
 	"github.com/google/triage-party/pkg/triage"
 	"github.com/google/triage-party/pkg/updater"
 )
 
 var (
 	// shared with tester
-	configPath      = flag.String("config", "", "configuration path")
-	initCachePath   = flag.String("initcache", "", "Where to load the initial cache from (optional)")
+	configPath     = flag.String("config", "", "configuration path")
+	persistBackend = flag.String("persist-backend", "", "Cache persistence backend (disk, mysql, cloudsql)")
+	persistPath    = flag.String("persist-path", "", "Where to persist cache to (automatic)")
+
 	reposOverride   = flag.String("repos", "", "Override configured repos with this repository (comma separated)")
 	githubTokenFile = flag.String("github-token-file", "", "github token secret file, also settable via GITHUB_TOKEN")
 
@@ -74,15 +87,13 @@ func main() {
 		klog.Exitf("open %s: %v", *configPath, err)
 	}
 
-	cachePath := *initCachePath
-	if cachePath == "" {
-		cachePath = initcache.DefaultDiskPath(*configPath, *reposOverride)
+	c, err := persist.FromEnv(*persistBackend, *persistPath, *configPath, *reposOverride)
+	if err != nil {
+		klog.Exitf("unable to create persistence layer: %v", err)
 	}
-	klog.Infof("cache path: %s", cachePath)
 
-	c := initcache.New(initcache.Config{Type: "disk", Path: cachePath})
 	if err := c.Initialize(); err != nil {
-		klog.Exitf("initcache load to %s: %v", cachePath, err)
+		klog.Exitf("persist init with %s: %v", c, err)
 	}
 
 	cfg := triage.Config{
@@ -111,11 +122,6 @@ func main() {
 		sn = calculateSiteName(ts)
 	}
 
-	// Make sure save works
-	if err := c.Save(); err != nil {
-		klog.Exitf("initcache save to %s: %v", cachePath, err)
-	}
-
 	u := updater.New(updater.Config{
 		Party:      tp,
 		MinRefresh: *minRefresh,
@@ -140,7 +146,7 @@ func main() {
 		for sig := range sigc {
 			klog.Infof("signal caught: %v", sig)
 			if err := c.Save(); err != nil {
-				klog.Errorf("save errro: %v", err)
+				klog.Errorf("unable to save: %v", err)
 			}
 			os.Exit(0)
 		}
@@ -156,7 +162,7 @@ func main() {
 		BaseDirectory: findPath(*siteDir),
 		Updater:       u,
 		Party:         tp,
-		WarnAge:       2 * *maxRefresh,
+		WarnAge:       *maxRefresh + 5*(time.Minute),
 		Name:          sn,
 	})
 

cmd/tester/main.go

@@ -23,7 +23,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/triage-party/pkg/initcache"
+	"github.com/google/triage-party/pkg/persist"
 	"github.com/google/triage-party/pkg/triage"
 
 	"github.com/google/go-github/v31/github"
@@ -32,9 +32,10 @@ import (
 )
 
 var (
-	// shared with tester
+	// shared with server
 	configPath      = flag.String("config", "", "configuration path")
-	initCachePath   = flag.String("initcache", "", "Where to load the initial cache from (optional)")
+	persistBackend  = flag.String("persist-backend", "", "Cache persistence backend (disk, mysql, cloudsql)")
+	persistPath     = flag.String("persist-path", "", "Where to persist cache to (automatic)")
 	reposOverride   = flag.String("repos", "", "Override configured repos with this repository (comma separated)")
 	githubTokenFile = flag.String("github-token-file", "", "github token secret file, also settable via GITHUB_TOKEN")
 
@@ -66,14 +67,13 @@ func main() {
 		klog.Exitf("open %s: %v", *configPath, err)
 	}
 
-	cachePath := *initCachePath
-	if cachePath == "" {
-		cachePath = initcache.DefaultDiskPath(*configPath, *reposOverride)
+	c, err := persist.FromEnv(*persistBackend, *persistPath, *configPath, *reposOverride)
+	if err != nil {
+		klog.Exitf("unable to create persistence layer: %v", err)
 	}
 
-	c := initcache.New(initcache.Config{Type: "disk", Path: cachePath})
 	if err := c.Initialize(); err != nil {
-		klog.Exitf("initcache load to %s: %v", cachePath, err)
+		klog.Exitf("persist initialize from %s: %v", c, err)
 	}
 
 	cfg := triage.Config{
@@ -99,7 +99,7 @@ func main() {
 	}
 
 	if err := c.Save(); err != nil {
-		klog.Exitf("initcache save to %s: %v", cachePath, err)
+		klog.Exitf("persist save to %s: %v", c, err)
 	}
 }
 

examples/minikube-deploy.sh

@@ -23,21 +23,16 @@ export IMAGE=gcr.io/k8s-minikube/triage-party
 export SERVICE_NAME=teaparty
 export CONFIG_FILE=examples/minikube.yaml
 
-env DOCKER_BUILDKIT=1 docker build \
-  -t "${IMAGE}" \
-  --build-arg "CFG=${CONFIG_FILE}" \
-  --secret "id=github,src=${GITHUB_TOKEN_PATH}" .
+docker build -t "${IMAGE}" --build-arg "CFG=${CONFIG_FILE}" .
 
 docker push "${IMAGE}" || exit 2
 
 readonly token="$(cat ${GITHUB_TOKEN_PATH})"
-
 gcloud beta run deploy "${SERVICE_NAME}" \
     --project "${PROJECT}" \
     --image "${IMAGE}" \
-    --set-env-vars="GITHUB_TOKEN=${token}" \
+    --set-env-vars="GITHUB_TOKEN=${token},PERSIST_BACKEND=cloudsql,PERSIST_PATH=tp:${DB_PASS}@tcp(k8s-minikube/us-central1/triage-party)/tp" \
     --allow-unauthenticated \
     --region us-central1 \
-    --max-instances 2 \
     --memory 384Mi \
     --platform managed

examples/skaffold-deploy.sh

@@ -23,21 +23,16 @@ export IMAGE="gcr.io/k8s-skaffold/teaparty:$(date +%F-%s)"
 export SERVICE_NAME=skaffold-triage-party
 export CONFIG_FILE=examples/skaffold.yaml
 
-env DOCKER_BUILDKIT=1 docker build \
-  -t "${IMAGE}" \
-  --build-arg "CFG=${CONFIG_FILE}" \
-  --secret "id=github,src=${GITHUB_TOKEN_PATH}" .
+docker build -t "${IMAGE}" --build-arg "CFG=${CONFIG_FILE}" .
 
 docker push "${IMAGE}" || exit 2
 
 readonly token="$(cat ${GITHUB_TOKEN_PATH})"
-
 gcloud beta run deploy "${SERVICE_NAME}" \
     --project "${PROJECT}" \
     --image "${IMAGE}" \
     --set-env-vars="GITHUB_TOKEN=${token}" \
     --allow-unauthenticated \
     --region us-central1 \
-    --max-instances 2 \
     --memory 384Mi \
     --platform managed

go.mod

@@ -3,14 +3,18 @@ module github.com/google/triage-party
 go 1.14
 
 require (
+	github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20200501161113-5e9e23d7cb91
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fatih/color v1.9.0 // indirect
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/google/go-github/v31 v31.0.0
 	github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e
 	github.com/imjasonmiller/godice v0.1.2
+	github.com/jmoiron/sqlx v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	gopkg.in/yaml.v2 v2.2.8
+	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.0.0
 )