WIP: APISnoop Spyglass Lens

prow/cmd/deck/BUILD.bazel

@@ -114,6 +114,7 @@ go_library(
         "//prow/spyglass/lenses/buildlog:go_default_library",
         "//prow/spyglass/lenses/junit:go_default_library",
         "//prow/spyglass/lenses/metadata:go_default_library",
+        "//prow/spyglass/lenses/apisnoop:go_default_library",
         "//prow/tide:go_default_library",
         "//prow/tide/history:go_default_library",
         "//vendor/cloud.google.com/go/storage:go_default_library",

prow/cmd/deck/main.go

@@ -66,6 +66,7 @@ import (
 	_ "k8s.io/test-infra/prow/spyglass/lenses/buildlog"
 	_ "k8s.io/test-infra/prow/spyglass/lenses/junit"
 	_ "k8s.io/test-infra/prow/spyglass/lenses/metadata"
+	_ "k8s.io/test-infra/prow/spyglass/lenses/apisnoop"
 )
 
 type options struct {

prow/cmd/deck/runlocal

@@ -25,4 +25,5 @@ curl "${HOST}/tide.js?var=tideData" > tide.js
 curl "${HOST}/tide-history.js?var=tideHistory" > tide-history.js
 curl "${HOST}/plugin-help.js?var=allHelp" > plugin-help.js
 curl "${HOST}/pr-data.js" > pr-data.js
-bazel run //prow/cmd/deck:deck -- --pregenerated-data=${DIR}/localdata --static-files-location=./prow/cmd/deck/static --template-files-location=./prow/cmd/deck/template --spyglass-files-location=./prow/spyglass/lenses --config-path ${DIR}/../../config.yaml --spyglass
+sed -i sXhttps://prow.k8s.ioXhttp://localhost:8080Xg *js
+bazel run //prow/cmd/deck:deck -- --pregenerated-data=${DIR}/localdata --static-files-location=./prow/cmd/deck/static --template-files-location=./prow/cmd/deck/template --spyglass-files-location=./prow/spyglass/lenses --config-path ${DIR}/../../config.yaml --spyglass

prow/config.yaml

@@ -1,7 +1,7 @@
 plank:
-  job_url_template: '{{if .Spec.Refs}}{{if eq .Spec.Refs.Org "kubernetes-security"}}https://console.cloud.google.com/storage/browser/kubernetes-security-prow/{{else}}https://prow.k8s.io/view/gcs/kubernetes-jenkins/{{end}}{{else}}https://prow.k8s.io/view/gcs/kubernetes-jenkins/{{end}}{{if eq .Spec.Type "presubmit"}}pr-logs/pull{{else if eq .Spec.Type "batch"}}pr-logs/pull{{else}}logs{{end}}{{if .Spec.Refs}}{{if ne .Spec.Refs.Org ""}}{{if ne .Spec.Refs.Org "kubernetes"}}/{{if and (eq .Spec.Refs.Org "kubernetes-sigs") (ne .Spec.Refs.Repo "poseidon")}}sigs.k8s.io{{else}}{{.Spec.Refs.Org}}{{end}}_{{.Spec.Refs.Repo}}{{else if ne .Spec.Refs.Repo "kubernetes"}}/{{.Spec.Refs.Repo}}{{end}}{{end}}{{end}}{{if eq .Spec.Type "presubmit"}}/{{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}}{{else if eq .Spec.Type "batch"}}/batch{{end}}/{{.Spec.Job}}/{{.Status.BuildID}}/'
-  report_template: '[Full PR test history](https://prow.k8s.io/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}}). [Your PR dashboard](https://gubernator.k8s.io/pr/{{with index .Spec.Refs.Pulls 0}}{{.Author}}{{end}}). Please help us cut down on flakes by [linking to](https://git.k8s.io/community/contributors/devel/flaky-tests.md#filing-issues-for-flaky-tests) an [open issue](https://github.com/{{.Spec.Refs.Org}}/{{.Spec.Refs.Repo}}/issues?q=is:issue+is:open) when you hit one in your PR.'
-  job_url_prefix: https://prow.k8s.io/view/gcs/
+  job_url_template: '{{if .Spec.Refs}}{{if eq .Spec.Refs.Org "kubernetes-security"}}https://console.cloud.google.com/storage/browser/kubernetes-security-prow/{{else}}http://localhost:8080/view/gcs/kubernetes-jenkins/{{end}}{{else}}http://localhost:8080/view/gcs/kubernetes-jenkins/{{end}}{{if eq .Spec.Type "presubmit"}}pr-logs/pull{{else if eq .Spec.Type "batch"}}pr-logs/pull{{else}}logs{{end}}{{if .Spec.Refs}}{{if ne .Spec.Refs.Org ""}}{{if ne .Spec.Refs.Org "kubernetes"}}/{{if and (eq .Spec.Refs.Org "kubernetes-sigs") (ne .Spec.Refs.Repo "poseidon")}}sigs.k8s.io{{else}}{{.Spec.Refs.Org}}{{end}}_{{.Spec.Refs.Repo}}{{else if ne .Spec.Refs.Repo "kubernetes"}}/{{.Spec.Refs.Repo}}{{end}}{{end}}{{end}}{{if eq .Spec.Type "presubmit"}}/{{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}}{{else if eq .Spec.Type "batch"}}/batch{{end}}/{{.Spec.Job}}/{{.Status.BuildID}}/'
+  report_template: '[Full PR test history](http://localhost:8080/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}}). [Your PR dashboard](https://gubernator.k8s.io/pr/{{with index .Spec.Refs.Pulls 0}}{{.Author}}{{end}}). Please help us cut down on flakes by [linking to](https://git.k8s.io/community/contributors/devel/flaky-tests.md#filing-issues-for-flaky-tests) an [open issue](https://github.com/{{.Spec.Refs.Org}}/{{.Spec.Refs.Repo}}/issues?q=is:issue+is:open) when you hit one in your PR.'
+  job_url_prefix: http://localhost:8080/view/gcs/
   pod_pending_timeout: 60m
   default_decoration_config:
     timeout: 7200000000000 # 2h
@@ -32,10 +32,14 @@ deck:
     viewers:
       "started.json|finished.json":
       - "metadata"
+      - "apisnoop"
       "build-log.txt":
       - "buildlog"
       "artifacts/junit.*\\.xml":
       - "junit"
+      # "endpoints.json":
+      # - "apisnoop"
+
     announcement: "The old job viewer, Gubernator, has been deprecated in favour of this page, Spyglass.{{if .ArtifactPath}} For now, the old page is <a href='https://gubernator.k8s.io/build/{{.ArtifactPath}}'>still available</a>.{{end}} Please send feedback to sig-testing."
   tide_update_period: 1s
   hidden_repos:
@@ -447,7 +451,7 @@ tide:
     kubernetes/dashboard: squash
     kubernetes/kube-deploy: squash
     kubernetes/website: squash
-  pr_status_base_url: https://prow.k8s.io/pr
+  pr_status_base_url: http://localhost:8080/pr
   blocker_label: tide/merge-blocker
   squash_label: tide/merge-method-squash
   rebase_label: tide/merge-method-rebase

prow/github/types.go

@@ -213,6 +213,7 @@ type PullRequestEvent struct {
 
 // PullRequest contains information about a PullRequest.
 type PullRequest struct {
+	ID                 int               `json:"id"`
 	Number             int               `json:"number"`
 	HTMLURL            string            `json:"html_url"`
 	User               User              `json:"user"`

prow/hook/BUILD.bazel

@@ -60,6 +60,7 @@ go_library(
         "//prow/plugins/owners-label:go_default_library",
         "//prow/plugins/pony:go_default_library",
         "//prow/plugins/project:go_default_library",
+        "//prow/plugins/projectmanager:go_default_library",
         "//prow/plugins/releasenote:go_default_library",
         "//prow/plugins/require-matching-label:go_default_library",
         "//prow/plugins/requiresig:go_default_library",

prow/hook/plugins.go

@@ -45,6 +45,7 @@ import (
 	_ "k8s.io/test-infra/prow/plugins/owners-label"
 	_ "k8s.io/test-infra/prow/plugins/pony"
 	_ "k8s.io/test-infra/prow/plugins/project"
+	_ "k8s.io/test-infra/prow/plugins/projectmanager"
 	_ "k8s.io/test-infra/prow/plugins/releasenote"
 	_ "k8s.io/test-infra/prow/plugins/require-matching-label"
 	_ "k8s.io/test-infra/prow/plugins/requiresig"

prow/plugins/BUILD.bazel

@@ -84,6 +84,7 @@ filegroup(
         "//prow/plugins/owners-label:all-srcs",
         "//prow/plugins/pony:all-srcs",
         "//prow/plugins/project:all-srcs",
+        "//prow/plugins/projectmanager:all-srcs",
         "//prow/plugins/releasenote:all-srcs",
         "//prow/plugins/require-matching-label:all-srcs",
         "//prow/plugins/requiresig:all-srcs",

prow/plugins/config.go

@@ -66,6 +66,7 @@ type Configuration struct {
 	Lgtm                       []Lgtm                 `json:"lgtm,omitempty"`
 	RepoMilestone              map[string]Milestone   `json:"repo_milestone,omitempty"`
 	Project                    ProjectConfig          `json:"project_config,omitempty"`
+	ProjectManager             ProjectManager         `json:"project_manager,omitempty"`
 	RequireMatchingLabel       []RequireMatchingLabel `json:"require_matching_label,omitempty"`
 	RequireSIG                 RequireSIG             `json:"requiresig,omitempty"`
 	Slack                      Slack                  `json:"slack,omitempty"`
@@ -437,6 +438,34 @@ type ProjectRepoConfig struct {
 	ProjectColumnMap map[string]string `json:"repo_default_column_map,omitempty"`
 }
 
+// ProjectManager represents the config for the ProjectManager plugin, holding top
+// level config options and a list of Projects
+type ProjectManager struct {
+	OrgRepos map[string]ManagedOrgRepo `json:"org/repos,omitempty"`
+}
+
+// ManagedOrgRepo is used by the ProjectManager plugin to represent an Organisation
+// or Repository with a list of Projects
+type ManagedOrgRepo struct {
+	Projects map[string]ManagedProject `json:"projects,omitempty"`
+}
+
+// ManagedProject is used by the ProjectManager plugin to represent a Project
+// with a list of Columns
+type ManagedProject struct {
+	Columns []ManagedColumn `json:"columns,omitempty"`
+}
+
+// ManagedColumn is used by the ProjectQueries plugin to represent a project column
+// and the conditions to add a PR to that column
+type ManagedColumn struct {
+	ID     int      `json:"id,omitempty"`
+	Name   string   `json:"name,omitempty"`
+	State  string   `json:"state,omitempty"`
+	Labels []string `json:"labels,omitempty"`
+	Org    string   `json:"org,omitempty"`
+}
+
 // MergeWarning is a config for the slackevents plugin's manual merge warnings.
 // If a PR is pushed to any of the repos listed in the config then send messages
 // to the all the slack channels listed if pusher is NOT in the whitelist.

prow/plugins/projectmanager/BUILD.bazel

@@ -0,0 +1,39 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["projectmanager.go"],
+    importpath = "k8s.io/test-infra/prow/plugins/projectmanager",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//prow/github:go_default_library",
+        "//prow/pluginhelp:go_default_library",
+        "//prow/plugins:go_default_library",
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["projectmanager_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//prow/github:go_default_library",
+        "//prow/plugins:go_default_library",
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

prow/plugins/projectmanager/projectmanager.go

@@ -0,0 +1,201 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package projectmanager is a plugin to auto add pull requests to project boards based on specified conditions
+package projectmanager
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"k8s.io/test-infra/prow/github"
+	"k8s.io/test-infra/prow/pluginhelp"
+	"k8s.io/test-infra/prow/plugins"
+)
+
+const (
+	pluginName = "project-manager"
+)
+
+// TODO Create a new handler for issues, look in hook/server.go
+func init() {
+	plugins.RegisterPullRequestHandler(pluginName, handlePullRequest, helpProvider)
+}
+
+func helpProvider(config *plugins.Configuration, enabledRepos []string) (*pluginhelp.PluginHelp, error) {
+	pluginHelp := &pluginhelp.PluginHelp{
+		Description: "The project-manager plugin automatically adds Pull Requests to specified GitHub Project Columns if they match given criteria.",
+		Config: func(config *plugins.Configuration) map[string]string {
+			configMap := make(map[string]string)
+			configString := "org/repos: {"
+			for orgRepoName, managedOrgRepo := range config.ProjectManager.OrgRepos {
+				configString := fmt.Sprintf("%s %s: { projects: {", configString, orgRepoName)
+				for projectName, managedProject := range managedOrgRepo.Projects {
+					configString := fmt.Sprintf("%s %s: { columns: [", configString, projectName)
+					for _, managedColumn := range managedProject.Columns {
+						configString := fmt.Sprintf("%s {id: \"%d\", name: \"%s\", state: \"%s\", org: \"%s\" labels: [", configString, managedColumn.ID, managedColumn.Name, managedColumn.State, managedColumn.Org)
+						for i, label := range managedColumn.Labels {
+							configString = fmt.Sprintf("%s \"%s\"", configString, label)
+							if i+1 < len(managedColumn.Labels) {
+								configString = fmt.Sprintf("%s,", configString)
+							}
+						}
+						configString = fmt.Sprintf("%s ] }", configString)
+					}
+					configString = fmt.Sprintf("%s ] }", configString)
+				}
+				configString = fmt.Sprintf("%s }", configString)
+			}
+			configString = fmt.Sprintf("%s }", configString)
+			configMap[""] = configString
+			return configMap
+		}(config),
+	}
+	return pluginHelp, nil
+}
+
+func handlePullRequest(pc plugins.Agent, pe github.PullRequestEvent) error {
+	return handlePR(pc.GitHubClient, pc.PluginConfig.ProjectManager, pc.Logger, pe)
+}
+
+// Strict subset of *github.Client methods.
+type githubClient interface {
+	GetIssueLabels(org, repo string, number int) ([]github.Label, error)
+	GetRepoProjects(owner, repo string) ([]github.Project, error)
+	GetOrgProjects(org string) ([]github.Project, error)
+	GetProjectColumns(projectID int) ([]github.ProjectColumn, error)
+	CreateProjectCard(columnID int, projectCard github.ProjectCard) (*github.ProjectCard, error)
+}
+
+func handlePR(gc githubClient, projectManager plugins.ProjectManager, log *logrus.Entry, pe github.PullRequestEvent) error {
+	// Only respond to label add or issue/PR open events
+	if pe.Action != github.PullRequestActionOpened &&
+		pe.Action != github.PullRequestActionReopened &&
+		pe.Action != github.PullRequestActionLabeled {
+		return nil
+	}
+	// Get any ManagedProjects that match this PR
+	matchedColumnIDs, err := getMatchingColumnIDs(gc, projectManager, pe)
+	if err != nil {
+		return err
+	}
+	// For each ManagedColumn that matches this PR, add this PR to that Project Column
+	for _, matchedColumnID := range matchedColumnIDs {
+		err = addPRToColumn(gc, matchedColumnID, pe)
+		log.WithError(err).Println("Failed to add PR to project")
+	}
+	return nil
+}
+
+func getMatchingColumnIDs(gc githubClient, projectManager plugins.ProjectManager, pe github.PullRequestEvent) ([]int, error) {
+	var matchedColumnIDs []int
+	// Don't use GetIssueLabels unless it's required and keep track of whether the labels have been fetched to avoid unnecessary API usage.
+	labelsFetched := false
+	var labels []github.Label
+	for orgRepoName, managedOrgRepo := range projectManager.OrgRepos {
+		for projectName, managedProject := range managedOrgRepo.Projects {
+			for _, managedColumn := range managedProject.Columns {
+				if managedColumn.Org != "" && managedColumn.Org != pe.Repo.Owner.Login {
+					continue
+				}
+				if managedColumn.State != "" && managedColumn.State != pe.PullRequest.State {
+					continue
+				}
+				if len(managedColumn.Labels) != 0 {
+					if !labelsFetched {
+						// If labels are not yet fetched then get them as they are now required
+						// GetIssueLabels works for PRs as they are considered issues in the API
+						var err error
+						labels, err = gc.GetIssueLabels(pe.Repo.Owner.Login, pe.Repo.Name, pe.Number)
+						if err != nil {
+							return nil, err
+						}
+						labelsFetched = true
+					}
+					if !hasLabels(managedColumn.Labels, labels) {
+						continue
+					}
+				}
+				columnID := managedColumn.ID
+				// Currently this assumes columnID having a value if 0 means it is unset
+				// While it's highly unlikely that an actual project would have an ID of 0, given that
+				// these IDs are global across GitHub, this doesn't seem like an ideal solution.
+				if columnID == 0 {
+					var err error
+					columnID, err = getColumnID(orgRepoName, projectName, managedColumn.Name, gc)
+					if err != nil {
+						return nil, err
+					}
+				}
+				matchedColumnIDs = append(matchedColumnIDs, columnID)
+			}
+		}
+	}
+	return matchedColumnIDs, nil
+}
+
+// hasLabels checks if all labels are in the github.label set "issueLabels"
+func hasLabels(labels []string, issueLabels []github.Label) bool {
+	for _, label := range labels {
+		if !github.HasLabel(label, issueLabels) {
+			return false
+		}
+	}
+	return true
+}
+
+func getColumnID(orgRepoName, projectName, columnName string, gc githubClient) (int, error) {
+	var projects []github.Project
+	var err error
+	orgRepoParts := strings.Split(orgRepoName, "/")
+	switch len(orgRepoParts) {
+	case 2:
+		projects, err = gc.GetRepoProjects(orgRepoParts[0], orgRepoParts[1])
+	case 1:
+		projects, err = gc.GetOrgProjects(orgRepoParts[0])
+	default:
+		return 0, fmt.Errorf("could not determine org or org/repo from %s", orgRepoName)
+	}
+	if err != nil {
+		return 0, err
+	}
+	for _, project := range projects {
+		if project.Name == projectName {
+			columns, err := gc.GetProjectColumns(project.ID)
+			if err != nil {
+				return 0, nil
+			}
+			for _, column := range columns {
+				if column.Name == columnName {
+					return column.ID, nil
+				}
+			}
+			return 0, fmt.Errorf("could not find column %s in project %s", columnName, projectName)
+		}
+	}
+	return 0, fmt.Errorf("could not find project %s in org/repo %s", projectName, orgRepoName)
+}
+
+func addPRToColumn(gc githubClient, columnID int, pe github.PullRequestEvent) error {
+	// Create project card and add this PR
+	projectCard := github.ProjectCard{}
+	projectCard.ContentType = "PullRequest"
+	projectCard.ContentID = pe.PullRequest.ID
+	_, err := gc.CreateProjectCard(columnID, projectCard)
+	if err != nil {
+		return err
+	}
+	return nil
+}