Skip to content

Commit

Permalink
refactor: build resource-group image in repo (#959)
Browse files Browse the repository at this point in the history
This changes the resource-group-controller image to be built alongside
the other first party images rather than built separately and imported
as a third party image. The ResourceGroup controller is a core component
of Config Sync, and this is intended to simplify the maintenance of this
component.
  • Loading branch information
sdowell authored Oct 23, 2023
1 parent 610b3b6 commit 4dbf320
Show file tree
Hide file tree
Showing 33 changed files with 3,424 additions and 10 deletions.
4 changes: 3 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ OCI_SYNC_IMAGE := oci-sync
HELM_SYNC_IMAGE := helm-sync
NOMOS_IMAGE := nomos
ASKPASS_IMAGE := gcenode-askpass-sidecar
RESOURCE_GROUP_IMAGE := resource-group-controller
# List of Config Sync images. Used to generate image-related variables/targets.
IMAGES := \
$(RECONCILER_IMAGE) \
Expand All @@ -135,7 +136,8 @@ IMAGES := \
$(OCI_SYNC_IMAGE) \
$(HELM_SYNC_IMAGE) \
$(NOMOS_IMAGE) \
$(ASKPASS_IMAGE)
$(ASKPASS_IMAGE) \
$(RESOURCE_GROUP_IMAGE)

# nomos binary for local run.
NOMOS_LOCAL := $(BIN_DIR)/linux_amd64/nomos
Expand Down
2 changes: 2 additions & 0 deletions Makefile.build
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ build-manifests-oss: "$(GOBIN)/addlicense" "$(BIN_DIR)/kustomize" $(OUTPUT_DIR)
-e "s|HYDRATION_CONTROLLER_IMAGE_NAME|$(call gen_image_tag,$(HYDRATION_CONTROLLER_IMAGE))|g" \
-e "s|RECONCILER_MANAGER_IMAGE_NAME|$(call gen_image_tag,$(RECONCILER_MANAGER_IMAGE))|g" \
-e "s|ASKPASS_IMAGE_NAME|$(call gen_image_tag,$(ASKPASS_IMAGE))|g" \
-e "s|RESOURCE_GROUP_CONTROLLER_IMAGE_NAME|$(call gen_image_tag,$(RESOURCE_GROUP_IMAGE))|g" \
> $(OSS_MANIFEST_STAGING_DIR)/config-sync-manifest.yaml
@ "$(GOBIN)/addlicense" $(OSS_MANIFEST_STAGING_DIR)/config-sync-manifest.yaml

Expand All @@ -176,6 +177,7 @@ build-manifests-operator: "$(GOBIN)/addlicense" "$(BIN_DIR)/kustomize" $(OUTPUT_
-e "s|RECONCILER_MANAGER_IMAGE_NAME|$(call gen_image_tag,$(RECONCILER_MANAGER_IMAGE))|g" \
-e "s|WEBHOOK_IMAGE_NAME|$(call gen_image_tag,$(ADMISSION_WEBHOOK_IMAGE))|g" \
-e "s|ASKPASS_IMAGE_NAME|$(call gen_image_tag,$(ASKPASS_IMAGE))|g" \
-e "s|RESOURCE_GROUP_CONTROLLER_IMAGE_NAME|$(call gen_image_tag,$(RESOURCE_GROUP_IMAGE))|g" \
> $(NOMOS_MANIFEST_STAGING_DIR)/config-sync-manifest.yaml
@ "$(GOBIN)/addlicense" $(NOMOS_MANIFEST_STAGING_DIR)/config-sync-manifest.yaml

Expand Down
13 changes: 12 additions & 1 deletion build/all/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on \
./cmd/admission-webhook \
./cmd/oci-sync \
./cmd/helm-sync \
./cmd/gcenode-askpass-sidecar
./cmd/gcenode-askpass-sidecar \
./cmd/resource-group


# Concatenate vendored licenses into LICENSES.txt
# Built in the container to include binary licenses (helm & kustomize)
Expand Down Expand Up @@ -142,6 +144,15 @@ COPY --from=bins /workspace/LICENSES.txt LICENSES.txt
USER nonroot:nonroot
ENTRYPOINT ["/gcenode-askpass-sidecar"]

# Resource group controller image
FROM gcr.io/distroless/static:nonroot as resource-group-controller
WORKDIR /
COPY --from=bins /go/bin/resource-group resource-group
COPY --from=bins /workspace/LICENSE LICENSE
COPY --from=bins /workspace/LICENSES.txt LICENSES.txt
USER nonroot:nonroot
ENTRYPOINT ["/resource-group"]

# Nomos image
# Not used by Config Sync backend components. Intended for use cases with the
# nomos CLI (e.g. containerized CI/CD)
Expand Down
25 changes: 25 additions & 0 deletions cmd/resource-group/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"os"

"kpt.dev/resourcegroup/controllers/runner"
)

func main() {
os.Exit(runner.Run())
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ require (
k8s.io/kubectl v0.26.9
k8s.io/kubernetes v1.26.9
k8s.io/utils v0.0.0-20230115233650-391b47cb4029
kpt.dev/resourcegroup v0.0.0-20221109031828-db4c3d2c630f
kpt.dev/resourcegroup v0.0.0-20231023223236-7ca71815022b
sigs.k8s.io/cli-utils v0.35.0
sigs.k8s.io/controller-runtime v0.14.1
sigs.k8s.io/kind v0.20.0
Expand Down
13 changes: 11 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,8 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-logr/glogr v1.2.2 h1:Jpc1ppf1yWnU+aRGFjIZeE2KfUnc1W4LV9XmndmP0QY=
github.com/go-logr/glogr v1.2.2/go.mod h1:u/37V9lMYDEmbMcbNNpRKnAB5Nof5FgtxhteHXbD3xY=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
Expand All @@ -149,6 +151,8 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
Expand Down Expand Up @@ -290,7 +294,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.7.0 h1:/XxtEV3I3Eif/HobnVx9YmJgk8ENdRsuUmM+fLCFNow=
github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE=
Expand Down Expand Up @@ -644,6 +651,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down Expand Up @@ -694,8 +703,8 @@ k8s.io/kubernetes v1.26.9 h1:vdTix+Rh3wbNvbXk/efOeDLX3lng12t1xdsG4rSksmk=
k8s.io/kubernetes v1.26.9/go.mod h1:gvP7bsbtu0/cA0ZBJqayLm9lS1PP3WCwrhQOAbpqsK8=
k8s.io/utils v0.0.0-20230115233650-391b47cb4029 h1:L8zDtT4jrxj+TaQYD0k8KNlr556WaVQylDXswKmX+dE=
k8s.io/utils v0.0.0-20230115233650-391b47cb4029/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
kpt.dev/resourcegroup v0.0.0-20221109031828-db4c3d2c630f h1:uHulkud4a4Z/Wbge4XuDGuywGHrl4l/fSpN8tVYJgBM=
kpt.dev/resourcegroup v0.0.0-20221109031828-db4c3d2c630f/go.mod h1:NUdYG1GBrhTRACE/TOH5Nv/Rf/rZ9ve3IvSJNhcJNfM=
kpt.dev/resourcegroup v0.0.0-20231023223236-7ca71815022b h1:+x8u8PkA7+VUZSw7adSLSzi9uCpsQOv7sUfgUCbsYA0=
kpt.dev/resourcegroup v0.0.0-20231023223236-7ca71815022b/go.mod h1:gRSeoBV3k0ecPbKU8Z8cWQ3JagHSXpNobLqV8h1SrD8=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
Expand Down
2 changes: 1 addition & 1 deletion manifests/base/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ resources:
- ../templates/otel-collector.yaml
- ../templates/reconciler-manager.yaml
- ../templates/reconciler-manager-configmap.yaml
- ../third_party/resourcegroup-manifest.yaml
- ../templates/resourcegroup-manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -495,11 +495,11 @@ spec:
- --metrics-addr=127.0.0.1:8080
- --enable-leader-election
command:
- /manager
- /resource-group
env:
- name: OC_RESOURCE_LABELS
value: k8s.container.name="manager"
image: gcr.io/config-management-release/resource-group-controller:v1.0.20
image: RESOURCE_GROUP_CONTROLLER_IMAGE_NAME
name: manager
resources:
requests:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handler

import (
"github.com/go-logr/logr"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/util/workqueue"
"kpt.dev/resourcegroup/apis/kpt.dev/v1alpha1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
)

// CRDEventHandler pushes an event to ResourceGroup event channel
// when the CRD or its CRs are contained in some ResourceGroup CRs.
type CRDEventHandler struct {
Mapping resourceMap
Channel chan event.GenericEvent
Log logr.Logger
}

var _ handler.EventHandler = &CRDEventHandler{}

// Create implements EventHandler
func (h *CRDEventHandler) Create(e event.CreateEvent, _ workqueue.RateLimitingInterface) {
h.Log.V(5).Info("received a create event")
h.enqueueEvent(e.Object)
}

// Update implements EventHandler
func (h *CRDEventHandler) Update(e event.UpdateEvent, _ workqueue.RateLimitingInterface) {
h.Log.V(5).Info("received an update event")
h.enqueueEvent(e.ObjectNew)
}

// Delete implements EventHandler
func (h *CRDEventHandler) Delete(e event.DeleteEvent, _ workqueue.RateLimitingInterface) {
h.Log.V(5).Info("received a delete event")
h.enqueueEvent(e.Object)
}

// Generic implements EventHandler
func (h *CRDEventHandler) Generic(e event.GenericEvent, _ workqueue.RateLimitingInterface) {
h.Log.V(5).Info("received a generic event")
h.enqueueEvent(e.Object)
}

func (h *CRDEventHandler) enqueueEvent(obj client.Object) {
crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition)
if !ok {
h.Log.Info("failed to derive a CRD from the event object", "name", obj.GetName())
return
}
gk := schema.GroupKind{
Group: crd.Spec.Group,
Kind: crd.Spec.Names.Kind,
}
for _, gknn := range h.Mapping.GetResources(gk) {
// clear the cached status for gknn since the CR status could change
// due to the change of CRD.
h.Log.V(5).Info("reset the cached resource status", "resource", gknn)
h.Mapping.SetStatus(gknn, nil)
for _, r := range h.Mapping.Get(gknn) {
var resgroup = &v1alpha1.ResourceGroup{}
resgroup.SetNamespace(r.Namespace)
resgroup.SetName(r.Name)
h.Log.V(5).Info("send a generic event for", "resourcegroup", resgroup.GetObjectMeta())
h.Channel <- event.GenericEvent{Object: resgroup}
}
}
}
102 changes: 102 additions & 0 deletions vendor/kpt.dev/resourcegroup/controllers/handler/event_handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handler

import (
"fmt"

"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/event"

"kpt.dev/resourcegroup/apis/kpt.dev/v1alpha1"
"kpt.dev/resourcegroup/controllers/resourcemap"
)

// resourceMap provides the interface for access the cached resources.
type resourceMap interface {
// Get maps from GKNN -> []RG. It gets the identifiers of ResourceGroup CRs
// that a GKNN is in.
Get(gknn v1alpha1.ObjMetadata) []types.NamespacedName
// GetResources map from GK -> []GKNN. It gets the list of GKNN for
// a given group kind.
GetResources(gk schema.GroupKind) []v1alpha1.ObjMetadata
// SetStatus sets the cached status for the given resource.
SetStatus(res v1alpha1.ObjMetadata, resStatus *resourcemap.CachedStatus)
}

// EnqueueEventToChannel pushes an event to ResourceGroup event channel
// instead of enqueue a Reqeust for ResourceGroup.
type EnqueueEventToChannel struct {
Mapping resourceMap
Channel chan event.GenericEvent
Log logr.Logger
GVK schema.GroupVersionKind
}

var _ cache.ResourceEventHandler = &EnqueueEventToChannel{}

// Create implements EventHandler
func (e *EnqueueEventToChannel) OnAdd(obj interface{}) {
e.Log.V(5).Info("received an add event")
e.enqueueEvent(obj)
}

// Update implements EventHandler
func (e *EnqueueEventToChannel) OnUpdate(_, newObj interface{}) {
e.Log.V(5).Info("received an update event")
e.enqueueEvent(newObj)
}

// Delete implements EventHandler
func (e *EnqueueEventToChannel) OnDelete(obj interface{}) {
e.Log.V(5).Info("received a delete event")
e.enqueueEvent(obj)
}

func (e *EnqueueEventToChannel) enqueueEvent(obj interface{}) {
gknn, err := e.toGKNN(obj)
if err != nil {
e.Log.Error(err, "failed to get GKNN from the received event", "object", obj)
return
}
for _, r := range e.Mapping.Get(gknn) {
var resgroup = &v1alpha1.ResourceGroup{}
resgroup.SetNamespace(r.Namespace)
resgroup.SetName(r.Name)
e.Log.V(5).Info("send a generic event for", "resourcegroup", resgroup.GetObjectMeta())
e.Channel <- event.GenericEvent{Object: resgroup}
}
}

func (e EnqueueEventToChannel) toGKNN(obj interface{}) (v1alpha1.ObjMetadata, error) {
metadata, err := meta.Accessor(obj)
if err != nil {
e.Log.Error(err, "missing object meta")
return v1alpha1.ObjMetadata{}, fmt.Errorf("missing object meta: %v", err)
}
gknn := v1alpha1.ObjMetadata{
Namespace: metadata.GetNamespace(),
Name: metadata.GetName(),
GroupKind: v1alpha1.GroupKind{
Group: e.GVK.Group,
Kind: e.GVK.Kind,
},
}
return gknn, nil
}
Loading

0 comments on commit 4dbf320

Please sign in to comment.