Merge pull request #32249 from cjwagner/image-build-split

Split Prow image publishing job to distinguish non-prow images.

.k8s-prow-images.yaml

@@ -0,0 +1,10 @@
+# The following images are published to gcr.io/k8s-prow even though they are not Prow images for legacy purposes.
+images:
+  - dir: label_sync
+  - dir: robots/commenter
+  - dir: robots/pr-creator
+  - dir: robots/issue-creator
+  - dir: testgrid/cmd/configurator
+  - dir: gcsweb/cmd/gcsweb
+  - dir: gencred
+  - dir: experiment/ml/analyze

Makefile

@@ -17,6 +17,9 @@
 # get the repo root and output path
 REPO_ROOT:=${CURDIR}
 OUT_DIR=$(REPO_ROOT)/_output
+# image building and publishing config
+REGISTRY ?= gcr.io/k8s-prow
+PROW_IMAGE ?=
 ################################################################################
 # ================================= Testing ====================================
 # unit tests (hermetic)
@@ -56,6 +59,27 @@ update-go-deps:
 .PHONY: verify-go-deps
 verify-go-deps:
 	hack/make-rules/verify/go-deps.sh
+# ======================== Image Building/Publishing ===========================
+# Build and publish miscellaneous images that get pushed to "gcr.io/k8s-prow/".
+# These are not prow images, but published there for legacy reasons.
+# The full set of images covered by these targets is configured in
+# .k8s-prow-images.yaml.
+.PHONY: push-misc-images
+push-misc-images:
+	./hack/make-rules/go-run/arbitrary.sh run ./hack/prowimagebuilder --prow-images-file=./.k8s-prow-images.yaml --ko-docker-repo="${REGISTRY}" --push=true
+
+.PHONY: build-misc-images
+build-misc-images:
+	./hack/make-rules/go-run/arbitrary.sh run ./hack/prowimagebuilder --prow-images-file=./.k8s-prow-images.yaml --ko-docker-repo="ko.local" --push=false
+
+.PHONY: push-single-image
+push-single-image:
+	./hack/make-rules/go-run/arbitrary.sh run ./hack/prowimagebuilder --prow-images-file=./.k8s-prow-images.yaml --ko-docker-repo="${REGISTRY}" --push=true --image=${PROW_IMAGE}
+
+.PHONY: build-single-image
+build-single-image:
+	./hack/make-rules/go-run/arbitrary.sh run ./hack/prowimagebuilder --prow-images-file=./.k8s-prow-images.yaml --ko-docker-repo="ko.local" --push=false --image=${PROW_IMAGE}
+
 ################################################################################
 # ================================== Linting ===================================
 # run linters, ensure generated code, etc.

config/jobs/kubernetes/test-infra/test-infra-presubmits.yaml

@@ -230,6 +230,39 @@ presubmits:
     annotations:
       testgrid-dashboards: presubmits-test-infra
       testgrid-tab-name: prow-image-build-test
+  # The following test builds the images specified in .k8s-prow-images.yaml
+  - name: pull-test-infra-misc-image-build-test
+    cluster: eks-prow-build-cluster
+    branches:
+    - master
+    # TODO: clean up this regexp once prow is published from kubernetes-sigs/prow.
+    run_if_changed: '^(\.ko\.yaml|hack/(ts-rollup|make-rules|prowimagebuilder)|prow|ghproxy|label_sync/.+\.go|robots/commenter|robots/pr-creator|robots/issue-creator|testgrid/cmd|gcsweb)'
+    decorate: true
+    labels:
+      # Building deck requires docker for typescript compilation.
+      preset-dind-enabled: "true"
+    spec:
+      containers:
+      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240310-bac0a3c105-test-infra
+        command:
+        - runner.sh
+        args:
+        - make
+        - build-misc-images
+        # docker-in-docker needs privileged mode
+        securityContext:
+          privileged: true
+        resources:
+          requests:
+            # This job is very CPU intensive as it builds images in parallel.
+            cpu: "14"
+            memory: "16Gi"
+          limits:
+            cpu: "14"
+            memory: "16Gi"
+    annotations:
+      testgrid-dashboards: presubmits-test-infra
+      testgrid-tab-name: misc-image-build-test
   - name: pull-test-infra-verify-lint
     cluster: eks-prow-build-cluster
     branches:

config/jobs/kubernetes/test-infra/test-infra-trusted.yaml

@@ -259,6 +259,48 @@ postsubmits:
     rerun_auth_config:
       github_users:
       - alvaroaleman
+  - name: post-test-infra-push-misc-images
+    cluster: test-infra-trusted
+    # TODO: clean up this regexp once prow is published from kubernetes-sigs/prow.
+    run_if_changed: '^(\.ko\.yaml|hack/(make-rules|prowimagebuilder)|gencred|prow|ghproxy|label_sync/.+\.go|robots/commenter|robots/pr-creator|robots/issue-creator|testgrid/cmd|gcsweb)'
+    decorate: true
+    labels:
+      # Building deck requires docker for typescript compilation.
+      preset-dind-enabled: "true"
+    branches:
+    - ^master$
+    max_concurrency: 1
+    spec:
+      serviceAccountName: pusher
+      containers:
+      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240310-bac0a3c105-test-infra
+        command:
+        - runner.sh
+        args:
+        - make
+        - push-misc-images
+        # docker-in-docker needs privileged mode
+        securityContext:
+          privileged: true
+        resources:
+          requests:
+            cpu: "15"
+      tolerations:
+      - key: "highcpu"
+        operator: "Equal"
+        value: "true"
+        effect: "NoSchedule"
+      nodeSelector:
+        highcpu: "true"
+    annotations:
+      testgrid-dashboards: sig-testing-prow
+      testgrid-tab-name: push-misc-images
+      testgrid-alert-email: kubernetes-sig-testing-alerts@googlegroups.com, k8s-infra-oncall@google.com
+      testgrid-num-failures-to-alert: '1'
+      description: builds and pushes misc test-infra images on each commit by running make push-misc-images
+    rerun_auth_config:
+      github_users:
+      - alvaroaleman
   - name: post-test-infra-push-kettle
     cluster: test-infra-trusted
     annotations:

prow/.prow-images.yaml

@@ -41,11 +41,3 @@ images:
   - dir: prow/external-plugins/cherrypicker
   - dir: prow/external-plugins/refresh
   - dir: ghproxy
-  - dir: label_sync
-  - dir: robots/commenter
-  - dir: robots/pr-creator
-  - dir: robots/issue-creator
-  - dir: testgrid/cmd/configurator
-  - dir: gcsweb/cmd/gcsweb
-  - dir: gencred
-  - dir: experiment/ml/analyze