From c8dcdd526aa9d1d1ce6e72e63caa0c355cf2f58e Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Wed, 10 Aug 2022 01:32:11 -0700 Subject: [PATCH] feat: add the leader election (#230) Co-authored-by: Ryan Zhang --- Makefile | 4 +- .../crds/multicluster.x-k8s.io_works.yaml | 413 +++++++++++------- charts/hub-agent/templates/deployment.yaml | 16 +- charts/hub-agent/templates/rbac.yaml | 2 +- .../multicluster.x-k8s.io_appliedworks.yaml | 192 ++++---- .../crds/multicluster.x-k8s.io_works.yaml | 187 -------- charts/member-agent/templates/deployment.yaml | 24 +- charts/member-agent/templates/rbac.yaml | 22 - cmd/hub-manager/app/controllermanager.go | 4 +- cmd/hubagent/main.go | 10 +- cmd/memberagent/main.go | 88 ++-- .../multicluster.x-k8s.io_appliedworks.yaml | 124 ++++++ .../bases/multicluster.x-k8s.io_works.yaml | 413 +++++++++++------- pkg/webhook/webhook.go | 2 +- test/e2e/framework/namespace.go | 2 +- 15 files changed, 816 insertions(+), 687 deletions(-) delete mode 100644 charts/member-agent/templates/crds/multicluster.x-k8s.io_works.yaml create mode 100644 config/crd/bases/multicluster.x-k8s.io_appliedworks.yaml diff --git a/Makefile b/Makefile index 8e8ec543e..8d8fde249 100644 --- a/Makefile +++ b/Makefile @@ -276,11 +276,11 @@ uninstall-helm-charts: clean-testing-kind-clusters-resources .PHONY: clean-testing-kind-clusters-resources clean-testing-kind-clusters-resources: kind export kubeconfig --name $(HUB_KIND_CLUSTER_NAME) - kubectl delete ns fleet-kind-member-testing --ignore-not-found + kubectl delete ns fleet-member-kind-member-testing --ignore-not-found kubectl delete memberclusters.fleet.azure.com kind-$(MEMBER_KIND_CLUSTER_NAME) --ignore-not-found kind export kubeconfig --name $(MEMBER_KIND_CLUSTER_NAME) - kubectl delete ns fleet-kind-member-testing --ignore-not-found + kubectl delete ns fleet-member-kind-member-testing --ignore-not-found .PHONY: clean-e2e-tests clean-e2e-tests: ## Remove diff --git a/charts/hub-agent/templates/crds/multicluster.x-k8s.io_works.yaml b/charts/hub-agent/templates/crds/multicluster.x-k8s.io_works.yaml index ecbdac888..dd4eb47d0 100644 --- a/charts/hub-agent/templates/crds/multicluster.x-k8s.io_works.yaml +++ b/charts/hub-agent/templates/crds/multicluster.x-k8s.io_works.yaml @@ -1,187 +1,264 @@ -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null name: works.multicluster.x-k8s.io spec: group: multicluster.x-k8s.io - scope: Namespaced names: + kind: Work + listKind: WorkList plural: works singular: work - kind: Work + scope: Namespaced versions: - - name: v1alpha1 - served: true - storage: true - subresources: - status: {} - "schema": - "openAPIV3Schema": - description: Work is the Schema for the works API - type: object - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: spec defines the workload of a work. - type: object - properties: - workload: - description: Workload represents the manifest workload to be deployed on spoke cluster + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Work is the Schema for the works API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: spec defines the workload of a work. + properties: + workload: + description: Workload represents the manifest workload to be deployed + on spoke cluster + properties: + manifests: + description: Manifests represents a list of kuberenetes resources + to be deployed on the spoke cluster. + items: + description: Manifest represents a resource to be deployed on + spoke cluster + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + type: object + status: + description: status defines the status of each applied manifest on the + spoke cluster. + properties: + conditions: + description: 'Conditions contains the different condition statuses + for this work. Valid condition types are: 1. Applied represents + workload in Work is applied successfully on the spoke cluster. 2. + Progressing represents workload in Work in the trasitioning from + one state to another the on the spoke cluster. 3. Available represents + workload in Work exists on the spoke cluster. 4. Degraded represents + the current state of workload does not match the desired state for + a certain period.' + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + type FooStatus struct{ // Represents the observations of a foo's + current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type type: object + type: array + manifestConditions: + description: ManifestConditions represents the conditions of each + resource in work deployed on spoke cluster. + items: + description: ManifestCondition represents the conditions of the + resources deployed on spoke cluster properties: - manifests: - description: Manifests represents a list of kuberenetes resources to be deployed on the spoke cluster. - type: array + conditions: + description: Conditions represents the conditions of this resource + on spoke cluster items: - description: Manifest represents a resource to be deployed on spoke cluster - type: object - x-kubernetes-preserve-unknown-fields: true - x-kubernetes-embedded-resource: true - status: - description: status defines the status of each applied manifest on the spoke cluster. - type: object - required: - - conditions - properties: - conditions: - description: 'Conditions contains the different condition statuses for this work. Valid condition types are: 1. Applied represents workload in Work is applied successfully on the spoke cluster. 2. Progressing represents workload in Work in the trasitioning from one state to another the on the spoke cluster. 3. Available represents workload in Work exists on the spoke cluster. 4. Degraded represents the current state of workload does not match the desired state for a certain period.' - type: array - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - type: object - required: - - lastTransitionTime - - message - - reason - - status - - type - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - type: string - format: date-time - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - type: string - maxLength: 32768 - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - type: integer - format: int64 - minimum: 0 - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: - description: status of the condition, one of True, False, Unknown. - type: string - enum: - - "True" - - "False" - - Unknown - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - manifestConditions: - description: ManifestConditions represents the conditions of each resource in work deployed on spoke cluster. - type: array - items: - description: ManifestCondition represents the conditions of the resources deployed on spoke cluster - type: object - required: - - conditions - properties: - conditions: - description: Conditions represents the conditions of this resource on spoke cluster - type: array - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - type: object - required: - - lastTransitionTime - - message - - reason - - status - - type - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - type: string - format: date-time - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - type: string - maxLength: 32768 - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - type: integer - format: int64 - minimum: 0 - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: - description: status of the condition, one of True, False, Unknown. - type: string - enum: - - "True" - - "False" - - Unknown - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - identifier: - description: resourceId represents a identity of a resource linking to manifests in spec. - type: object + description: "Condition contains details for one aspect of + the current state of this API Resource. --- This struct + is intended for direct use as an array at the field path + .status.conditions. For example, type FooStatus struct{ + // Represents the observations of a foo's current state. + // Known .status.conditions.type are: \"Available\", \"Progressing\", + and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields + }" properties: - group: - description: Group is the group of the resource. - type: string - kind: - description: Kind is the kind of the resource. + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should + be when the underlying condition changed. If that is + not known, then using the time when the API field changed + is acceptable. + format: date-time type: string - name: - description: Name is the name of the resource + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 type: string - namespace: - description: Namespace is the namespace of the resource, the resource is cluster scoped if the value is empty - type: string - ordinal: - description: Ordinal represents an index in manifests list, so the condition can still be linked to a manifest even thougth manifest cannot be parsed successfully. + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 type: integer - resource: - description: Resource is the resource type of the resource + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last transition. + Producers of specific condition types may define expected + values and meanings for this field, and whether the + values are considered a guaranteed API. The value should + be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown type: string - version: - description: Version is the version of the resource. + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across + resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability + to deconflict is important. The regex it matches is + (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + identifier: + description: resourceId represents a identity of a resource + linking to manifests in spec. + properties: + group: + description: Group is the group of the resource. + type: string + kind: + description: Kind is the kind of the resource. + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: Namespace is the namespace of the resource, + the resource is cluster scoped if the value is empty + type: string + ordinal: + description: Ordinal represents an index in manifests list, + so the condition can still be linked to a manifest even + thougth manifest cannot be parsed successfully. + type: integer + resource: + description: Resource is the resource type of the resource + type: string + version: + description: Version is the version of the resource. + type: string + required: + - ordinal + type: object + required: + - conditions + type: object + type: array + required: + - conditions + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/hub-agent/templates/deployment.yaml b/charts/hub-agent/templates/deployment.yaml index d99c7b9df..b7be0c89b 100644 --- a/charts/hub-agent/templates/deployment.yaml +++ b/charts/hub-agent/templates/deployment.yaml @@ -23,10 +23,22 @@ spec: - --leader-elect=true - --enable-webhook={{ .Values.enableWebhook }} - --v={{ .Values.logVerbosity }} + - -add_dir_header ports: - - name: http - containerPort: 80 + - name: metrics + containerPort: 8080 protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + readinessProbe: + httpGet: + path: /readyz + port: healthz env: - name: POD_NAMESPACE valueFrom: diff --git a/charts/hub-agent/templates/rbac.yaml b/charts/hub-agent/templates/rbac.yaml index 88195f615..beb0f3431 100644 --- a/charts/hub-agent/templates/rbac.yaml +++ b/charts/hub-agent/templates/rbac.yaml @@ -1,7 +1,7 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: hub-agent-role-binding + name: {{ include "hub-agent.fullname" . }}-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/charts/member-agent/templates/crds/multicluster.x-k8s.io_appliedworks.yaml b/charts/member-agent/templates/crds/multicluster.x-k8s.io_appliedworks.yaml index e4df0539d..19105743f 100644 --- a/charts/member-agent/templates/crds/multicluster.x-k8s.io_appliedworks.yaml +++ b/charts/member-agent/templates/crds/multicluster.x-k8s.io_appliedworks.yaml @@ -1,94 +1,124 @@ -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null name: appliedworks.multicluster.x-k8s.io spec: group: multicluster.x-k8s.io names: + categories: + - fleet kind: AppliedWork listKind: AppliedWorkList plural: appliedworks singular: appliedwork scope: Cluster versions: - - name: v1alpha1 - served: true - storage: true - subresources: - status: {} - "schema": - "openAPIV3Schema": - description: AppliedWork represents an applied work on managed cluster that is placed on a managed cluster. An appliedwork links to a work on a hub recording resources deployed in the managed cluster. When the agent is removed from managed cluster, cluster-admin on managed cluster can delete appliedmanifestwork to remove resources deployed by the agent. The name of the appliedwork must be the same as {manifestwork name} The namespace of the appliedwork should be the same as the resource applied on the managed cluster. - type: object - required: - - spec - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Spec represents the desired configuration of AppliedManifestWork. - type: object - required: - - workName - - workNamespace - properties: - workName: - description: WorkName represents the name of the related work on the hub. - type: string - workNamespace: - description: WorkNamespace represents the namespace of the related work on the hub. - type: string - status: - description: Status represents the current status of AppliedManifestWork. - type: object - properties: - appliedResources: - description: AppliedResources represents a list of resources defined within the manifestwork that are applied. Only resources with valid GroupVersionResource, namespace, and name are suitable. An item in this slice is deleted when there is no mapped manifest in manifestwork.Spec or by finalizer. The resource relating to the item will also be removed from managed cluster. The deleted resource may still be present until the finalizers for that resource are finished. However, the resource will not be undeleted, so it can be removed from this list and eventual consistency is preserved. - type: array - items: - description: AppliedResourceMeta represents the group, version, resource, name and namespace of a resource. Since these resources have been created, they must have valid group, version, resource, namespace, and name. - type: object - properties: - group: - description: Group is the group of the resource. - type: string - kind: - description: Kind is the kind of the resource. - type: string - name: - description: Name is the name of the resource - type: string - namespace: - description: Namespace is the namespace of the resource, the resource is cluster scoped if the value is empty - type: string - ordinal: - description: Ordinal represents an index in manifests list, so the condition can still be linked to a manifest even thougth manifest cannot be parsed successfully. - type: integer - resource: - description: Resource is the resource type of the resource - type: string - uid: - description: UID is set on successful deletion of the Kubernetes resource by controller. The resource might be still visible on the managed cluster after this field is set. It is not directly settable by a client. - type: string - version: - description: Version is the version of the resource. - type: string \ No newline at end of file + - name: v1alpha1 + schema: + openAPIV3Schema: + description: AppliedWork represents an applied work on managed cluster that + is placed on a managed cluster. An appliedwork links to a work on a hub + recording resources deployed in the managed cluster. When the agent is removed + from managed cluster, cluster-admin on managed cluster can delete appliedmanifestwork + to remove resources deployed by the agent. The name of the appliedwork must + be the same as {manifestwork name} The namespace of the appliedwork should + be the same as the resource applied on the managed cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec represents the desired configuration of AppliedManifestWork. + properties: + workName: + description: WorkName represents the name of the related work on the + hub. + type: string + workNamespace: + description: WorkNamespace represents the namespace of the related + work on the hub. + type: string + required: + - workName + - workNamespace + type: object + status: + description: Status represents the current status of AppliedManifestWork. + properties: + appliedResources: + description: AppliedResources represents a list of resources defined + within the manifestwork that are applied. Only resources with valid + GroupVersionResource, namespace, and name are suitable. An item + in this slice is deleted when there is no mapped manifest in manifestwork.Spec + or by finalizer. The resource relating to the item will also be + removed from managed cluster. The deleted resource may still be + present until the finalizers for that resource are finished. However, + the resource will not be undeleted, so it can be removed from this + list and eventual consistency is preserved. + items: + description: AppliedResourceMeta represents the group, version, + resource, name and namespace of a resource. Since these resources + have been created, they must have valid group, version, resource, + namespace, and name. + properties: + group: + description: Group is the group of the resource. + type: string + kind: + description: Kind is the kind of the resource. + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: Namespace is the namespace of the resource, the + resource is cluster scoped if the value is empty + type: string + ordinal: + description: Ordinal represents an index in manifests list, + so the condition can still be linked to a manifest even thougth + manifest cannot be parsed successfully. + type: integer + resource: + description: Resource is the resource type of the resource + type: string + uid: + description: UID is set on successful deletion of the Kubernetes + resource by controller. The resource might be still visible + on the managed cluster after this field is set. It is not + directly settable by a client. + type: string + version: + description: Version is the version of the resource. + type: string + required: + - ordinal + type: object + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/member-agent/templates/crds/multicluster.x-k8s.io_works.yaml b/charts/member-agent/templates/crds/multicluster.x-k8s.io_works.yaml deleted file mode 100644 index ecbdac888..000000000 --- a/charts/member-agent/templates/crds/multicluster.x-k8s.io_works.yaml +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: works.multicluster.x-k8s.io -spec: - group: multicluster.x-k8s.io - scope: Namespaced - names: - plural: works - singular: work - kind: Work - versions: - - name: v1alpha1 - served: true - storage: true - subresources: - status: {} - "schema": - "openAPIV3Schema": - description: Work is the Schema for the works API - type: object - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: spec defines the workload of a work. - type: object - properties: - workload: - description: Workload represents the manifest workload to be deployed on spoke cluster - type: object - properties: - manifests: - description: Manifests represents a list of kuberenetes resources to be deployed on the spoke cluster. - type: array - items: - description: Manifest represents a resource to be deployed on spoke cluster - type: object - x-kubernetes-preserve-unknown-fields: true - x-kubernetes-embedded-resource: true - status: - description: status defines the status of each applied manifest on the spoke cluster. - type: object - required: - - conditions - properties: - conditions: - description: 'Conditions contains the different condition statuses for this work. Valid condition types are: 1. Applied represents workload in Work is applied successfully on the spoke cluster. 2. Progressing represents workload in Work in the trasitioning from one state to another the on the spoke cluster. 3. Available represents workload in Work exists on the spoke cluster. 4. Degraded represents the current state of workload does not match the desired state for a certain period.' - type: array - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - type: object - required: - - lastTransitionTime - - message - - reason - - status - - type - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - type: string - format: date-time - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - type: string - maxLength: 32768 - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - type: integer - format: int64 - minimum: 0 - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: - description: status of the condition, one of True, False, Unknown. - type: string - enum: - - "True" - - "False" - - Unknown - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - manifestConditions: - description: ManifestConditions represents the conditions of each resource in work deployed on spoke cluster. - type: array - items: - description: ManifestCondition represents the conditions of the resources deployed on spoke cluster - type: object - required: - - conditions - properties: - conditions: - description: Conditions represents the conditions of this resource on spoke cluster - type: array - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - type: object - required: - - lastTransitionTime - - message - - reason - - status - - type - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - type: string - format: date-time - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - type: string - maxLength: 32768 - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - type: integer - format: int64 - minimum: 0 - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: - description: status of the condition, one of True, False, Unknown. - type: string - enum: - - "True" - - "False" - - Unknown - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - identifier: - description: resourceId represents a identity of a resource linking to manifests in spec. - type: object - properties: - group: - description: Group is the group of the resource. - type: string - kind: - description: Kind is the kind of the resource. - type: string - name: - description: Name is the name of the resource - type: string - namespace: - description: Namespace is the namespace of the resource, the resource is cluster scoped if the value is empty - type: string - ordinal: - description: Ordinal represents an index in manifests list, so the condition can still be linked to a manifest even thougth manifest cannot be parsed successfully. - type: integer - resource: - description: Resource is the resource type of the resource - type: string - version: - description: Version is the version of the resource. - type: string diff --git a/charts/member-agent/templates/deployment.yaml b/charts/member-agent/templates/deployment.yaml index 6a4f5f2fc..cfa1a1468 100644 --- a/charts/member-agent/templates/deployment.yaml +++ b/charts/member-agent/templates/deployment.yaml @@ -24,8 +24,10 @@ spec: - name: http containerPort: 80 args: + - --leader-elect=true - --tls-insecure={{ .Values.tlsClientInsecure }} - --v={{ .Values.logVerbosity }} + - -add_dir_header env: - name: HUB_SERVER_URL value: "{{ .Values.config.hubURL }}" @@ -37,10 +39,30 @@ spec: value: "{{ .Values.config.hubCA }}" resources: {{- toYaml .Values.resources | nindent 12 }} + ports: + - containerPort: 8080 + name: hubmetrics + protocol: TCP + - containerPort: 8081 + name: hubhealthz + protocol: TCP + - containerPort: 8090 + name: membermetrics + protocol: TCP + - containerPort: 8091 + name: memberhealthz + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: hubhealthz + readinessProbe: + httpGet: + path: /readyz + port: hubhealthz volumeMounts: - name: provider-token mountPath: /config - - name: refresh-token image: "{{ .Values.refreshtoken.repository }}:{{ .Values.refreshtoken.tag }}" imagePullPolicy: {{ .Values.refreshtoken.pullPolicy }} diff --git a/charts/member-agent/templates/rbac.yaml b/charts/member-agent/templates/rbac.yaml index 37aaa0dea..936a9c4dc 100644 --- a/charts/member-agent/templates/rbac.yaml +++ b/charts/member-agent/templates/rbac.yaml @@ -1,12 +1,3 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "member-agent.fullname" . }}-member-agent -rules: - - apiGroups: [ "multicluster.x-k8s.io" ] - resources: [ "*" ] - verbs: [ "*" ] ---- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -19,16 +10,3 @@ subjects: - kind: ServiceAccount name: {{ include "member-agent.fullname" . }}-sa namespace: {{.Values.namespace}} ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "member-agent.fullname" . }}-member-agent-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "member-agent.fullname" . }}-member-agent -subjects: - - kind: ServiceAccount - name: {{ include "member-agent.fullname" . }}-sa - namespace: {{.Values.namespace}} diff --git a/cmd/hub-manager/app/controllermanager.go b/cmd/hub-manager/app/controllermanager.go index c7d6ab3a4..9f476bc1c 100644 --- a/cmd/hub-manager/app/controllermanager.go +++ b/cmd/hub-manager/app/controllermanager.go @@ -110,12 +110,12 @@ func Run(ctx context.Context, opts *options.Options) error { return err } // TODO: add real healthy check - if err = mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { klog.ErrorS(err, "failed to add health check endpoint") return err } // TODO: add real ready check - if err = mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { klog.ErrorS(err, "failed to add ready check endpoint") return err } diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go index b523bb49a..17fb80dfc 100644 --- a/cmd/hubagent/main.go +++ b/cmd/hubagent/main.go @@ -32,14 +32,14 @@ var ( probeAddr = flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") metricsAddr = flag.String("metrics-bind-address", ":8080", "The address the metric endpoint binds to.") enableLeaderElection = flag.Bool("leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + leaderElectionNamespace = flag.String("leader-election-namespace", "kube-system", "The namespace in which the leader election resource will be created.") enableWebhook = flag.Bool("enable-webhook", false, "If set, the fleet webhook is enabled.") networkingAgentsEnabled = flag.Bool("networking-agents-enabled", false, "Whether the networking agents are enabled or not.") ) const ( - FleetWebhookCertDir = "/tmp/k8s-webhook-server/serving-certs" - FleetWebhookPort = 9443 - LeaderElectionNamespace = "kube-system" + FleetWebhookCertDir = "/tmp/k8s-webhook-server/serving-certs" + FleetWebhookPort = 9443 ) func init() { @@ -65,8 +65,8 @@ func main() { CertDir: FleetWebhookCertDir, HealthProbeBindAddress: *probeAddr, LeaderElection: *enableLeaderElection, - LeaderElectionNamespace: LeaderElectionNamespace, - LeaderElectionID: "984738fa.hub.fleet.azure.com", + LeaderElectionNamespace: *leaderElectionNamespace, + LeaderElectionID: "13622se4848560.hub.fleet.azure.com", }) if err != nil { klog.ErrorS(err, "unable to start controller manager.") diff --git a/cmd/memberagent/main.go b/cmd/memberagent/main.go index a8b13d3aa..eab8292bd 100644 --- a/cmd/memberagent/main.go +++ b/cmd/memberagent/main.go @@ -21,7 +21,6 @@ import ( "k8s.io/client-go/util/retry" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -40,10 +39,11 @@ var ( tlsClientInsecure = flag.Bool("tls-insecure", false, "Enable TLSClientConfig.Insecure property. Enabling this will make the connection inSecure (should be 'true' for testing purpose only.)") hubProbeAddr = flag.String("hub-health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") hubMetricsAddr = flag.String("hub-metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - probeAddr = flag.String("health-probe-bind-address", ":8082", "The address the probe endpoint binds to.") + probeAddr = flag.String("health-probe-bind-address", ":8091", "The address the probe endpoint binds to.") metricsAddr = flag.String("metrics-bind-address", ":8090", "The address the metric endpoint binds to.") enableLeaderElection = flag.Bool("leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + leaderElectionNamespace = flag.String("leader-election-namespace", "kube-system", "The namespace in which the leader election resource will be created.") ) func init() { @@ -126,45 +126,64 @@ func main() { } hubOpts := ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: *hubMetricsAddr, - Port: 8443, - HealthProbeBindAddress: *hubProbeAddr, - LeaderElection: *enableLeaderElection, - LeaderElectionID: "984738fa.hub.fleet.azure.com", - Namespace: mcNamespace, + Scheme: scheme, + MetricsBindAddress: *hubMetricsAddr, + Port: 8443, + HealthProbeBindAddress: *hubProbeAddr, + LeaderElection: *enableLeaderElection, + LeaderElectionNamespace: mcNamespace, // This requires we have access to resource "leases" in API group "coordination.k8s.io" under namespace $mcHubNamespace + LeaderElectionID: "136224848560.hub.fleet.azure.com", + Namespace: mcNamespace, } + memberOpts := ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: *metricsAddr, + Port: 9443, + HealthProbeBindAddress: *probeAddr, + LeaderElection: hubOpts.LeaderElection, + LeaderElectionNamespace: *leaderElectionNamespace, + LeaderElectionID: "136224848560.member.fleet.azure.com", + } //+kubebuilder:scaffold:builder - if err := Start(ctrl.SetupSignalHandler(), &hubConfig, hubOpts); err != nil { + if err := Start(ctrl.SetupSignalHandler(), &hubConfig, hubOpts, memberOpts); err != nil { klog.ErrorS(err, "problem running controllers") os.Exit(1) } } // Start the member controllers with the supplied config -func Start(ctx context.Context, hubCfg *rest.Config, hubOpts ctrl.Options) error { +func Start(ctx context.Context, hubCfg *rest.Config, hubOpts, memberOpts ctrl.Options) error { hubMgr, err := ctrl.NewManager(hubCfg, hubOpts) if err != nil { return errors.Wrap(err, "unable to start hub manager") } - memberOpts := ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: *metricsAddr, - Port: 8446, - HealthProbeBindAddress: *probeAddr, - LeaderElection: hubOpts.LeaderElection, - LeaderElectionID: "984738fa.member.fleet.azure.com", - } - memberConfig := ctrl.GetConfigOrDie() memberMgr, err := ctrl.NewManager(memberConfig, memberOpts) if err != nil { return errors.Wrap(err, "unable to start member manager") } + if err := hubMgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + klog.ErrorS(err, "unable to set up health check for hub manager") + os.Exit(1) + } + if err := hubMgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + klog.ErrorS(err, "unable to set up ready check for hub manager") + os.Exit(1) + } + + if err := memberMgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + klog.ErrorS(err, "unable to set up health check for member manager") + os.Exit(1) + } + if err := memberMgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + klog.ErrorS(err, "unable to set up ready check for member manager") + os.Exit(1) + } + spokeDynamicClient, err := dynamic.NewForConfig(memberConfig) if err != nil { klog.ErrorS(err, "unable to create spoke dynamic client") @@ -177,10 +196,6 @@ func Start(ctx context.Context, hubCfg *rest.Config, hubOpts ctrl.Options) error os.Exit(1) } - spokeClient, err := client.New(memberConfig, client.Options{ - Scheme: memberOpts.Scheme, Mapper: restMapper, - }) - if err != nil { klog.ErrorS(err, "unable to create spoke client") os.Exit(1) @@ -189,7 +204,7 @@ func Start(ctx context.Context, hubCfg *rest.Config, hubOpts ctrl.Options) error if err = workcontrollers.NewWorkStatusReconciler( hubMgr.GetClient(), spokeDynamicClient, - spokeClient, + memberMgr.GetClient(), restMapper, hubMgr.GetEventRecorderFor("work_status_controller"), 3, @@ -201,7 +216,7 @@ func Start(ctx context.Context, hubCfg *rest.Config, hubOpts ctrl.Options) error if err = workcontrollers.NewApplyWorkReconciler( hubMgr.GetClient(), spokeDynamicClient, - spokeClient, + memberMgr.GetClient(), restMapper, hubMgr.GetEventRecorderFor("work_controller"), 3, @@ -212,7 +227,7 @@ func Start(ctx context.Context, hubCfg *rest.Config, hubOpts ctrl.Options) error if err = workcontrollers.NewFinalizeWorkReconciler( hubMgr.GetClient(), - spokeClient, + memberMgr.GetClient(), hubMgr.GetEventRecorderFor("WorkFinalizer_controller"), ).SetupWithManager(hubMgr); err != nil { klog.ErrorS(err, "unable to create controller", "controller", "WorkFinalize") @@ -223,25 +238,6 @@ func Start(ctx context.Context, hubCfg *rest.Config, hubOpts ctrl.Options) error return errors.Wrap(err, "unable to create controller hub_member") } - if err := hubMgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - klog.ErrorS(err, "unable to set up health check for hub manager") - os.Exit(1) - } - if err := hubMgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - klog.ErrorS(err, "unable to set up ready check for hub manager") - os.Exit(1) - } - - if err := memberMgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - klog.ErrorS(err, "unable to set up health check for member manager") - os.Exit(1) - } - if err := memberMgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - klog.ErrorS(err, "unable to set up ready check for member manager") - - os.Exit(1) - } - klog.V(3).InfoS("starting hub manager") startErr := make(chan error) go func() { diff --git a/config/crd/bases/multicluster.x-k8s.io_appliedworks.yaml b/config/crd/bases/multicluster.x-k8s.io_appliedworks.yaml new file mode 100644 index 000000000..19105743f --- /dev/null +++ b/config/crd/bases/multicluster.x-k8s.io_appliedworks.yaml @@ -0,0 +1,124 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: appliedworks.multicluster.x-k8s.io +spec: + group: multicluster.x-k8s.io + names: + categories: + - fleet + kind: AppliedWork + listKind: AppliedWorkList + plural: appliedworks + singular: appliedwork + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: AppliedWork represents an applied work on managed cluster that + is placed on a managed cluster. An appliedwork links to a work on a hub + recording resources deployed in the managed cluster. When the agent is removed + from managed cluster, cluster-admin on managed cluster can delete appliedmanifestwork + to remove resources deployed by the agent. The name of the appliedwork must + be the same as {manifestwork name} The namespace of the appliedwork should + be the same as the resource applied on the managed cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec represents the desired configuration of AppliedManifestWork. + properties: + workName: + description: WorkName represents the name of the related work on the + hub. + type: string + workNamespace: + description: WorkNamespace represents the namespace of the related + work on the hub. + type: string + required: + - workName + - workNamespace + type: object + status: + description: Status represents the current status of AppliedManifestWork. + properties: + appliedResources: + description: AppliedResources represents a list of resources defined + within the manifestwork that are applied. Only resources with valid + GroupVersionResource, namespace, and name are suitable. An item + in this slice is deleted when there is no mapped manifest in manifestwork.Spec + or by finalizer. The resource relating to the item will also be + removed from managed cluster. The deleted resource may still be + present until the finalizers for that resource are finished. However, + the resource will not be undeleted, so it can be removed from this + list and eventual consistency is preserved. + items: + description: AppliedResourceMeta represents the group, version, + resource, name and namespace of a resource. Since these resources + have been created, they must have valid group, version, resource, + namespace, and name. + properties: + group: + description: Group is the group of the resource. + type: string + kind: + description: Kind is the kind of the resource. + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: Namespace is the namespace of the resource, the + resource is cluster scoped if the value is empty + type: string + ordinal: + description: Ordinal represents an index in manifests list, + so the condition can still be linked to a manifest even thougth + manifest cannot be parsed successfully. + type: integer + resource: + description: Resource is the resource type of the resource + type: string + uid: + description: UID is set on successful deletion of the Kubernetes + resource by controller. The resource might be still visible + on the managed cluster after this field is set. It is not + directly settable by a client. + type: string + version: + description: Version is the version of the resource. + type: string + required: + - ordinal + type: object + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/multicluster.x-k8s.io_works.yaml b/config/crd/bases/multicluster.x-k8s.io_works.yaml index ecbdac888..dd4eb47d0 100644 --- a/config/crd/bases/multicluster.x-k8s.io_works.yaml +++ b/config/crd/bases/multicluster.x-k8s.io_works.yaml @@ -1,187 +1,264 @@ -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null name: works.multicluster.x-k8s.io spec: group: multicluster.x-k8s.io - scope: Namespaced names: + kind: Work + listKind: WorkList plural: works singular: work - kind: Work + scope: Namespaced versions: - - name: v1alpha1 - served: true - storage: true - subresources: - status: {} - "schema": - "openAPIV3Schema": - description: Work is the Schema for the works API - type: object - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: spec defines the workload of a work. - type: object - properties: - workload: - description: Workload represents the manifest workload to be deployed on spoke cluster + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Work is the Schema for the works API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: spec defines the workload of a work. + properties: + workload: + description: Workload represents the manifest workload to be deployed + on spoke cluster + properties: + manifests: + description: Manifests represents a list of kuberenetes resources + to be deployed on the spoke cluster. + items: + description: Manifest represents a resource to be deployed on + spoke cluster + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + type: object + status: + description: status defines the status of each applied manifest on the + spoke cluster. + properties: + conditions: + description: 'Conditions contains the different condition statuses + for this work. Valid condition types are: 1. Applied represents + workload in Work is applied successfully on the spoke cluster. 2. + Progressing represents workload in Work in the trasitioning from + one state to another the on the spoke cluster. 3. Available represents + workload in Work exists on the spoke cluster. 4. Degraded represents + the current state of workload does not match the desired state for + a certain period.' + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + type FooStatus struct{ // Represents the observations of a foo's + current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type type: object + type: array + manifestConditions: + description: ManifestConditions represents the conditions of each + resource in work deployed on spoke cluster. + items: + description: ManifestCondition represents the conditions of the + resources deployed on spoke cluster properties: - manifests: - description: Manifests represents a list of kuberenetes resources to be deployed on the spoke cluster. - type: array + conditions: + description: Conditions represents the conditions of this resource + on spoke cluster items: - description: Manifest represents a resource to be deployed on spoke cluster - type: object - x-kubernetes-preserve-unknown-fields: true - x-kubernetes-embedded-resource: true - status: - description: status defines the status of each applied manifest on the spoke cluster. - type: object - required: - - conditions - properties: - conditions: - description: 'Conditions contains the different condition statuses for this work. Valid condition types are: 1. Applied represents workload in Work is applied successfully on the spoke cluster. 2. Progressing represents workload in Work in the trasitioning from one state to another the on the spoke cluster. 3. Available represents workload in Work exists on the spoke cluster. 4. Degraded represents the current state of workload does not match the desired state for a certain period.' - type: array - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - type: object - required: - - lastTransitionTime - - message - - reason - - status - - type - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - type: string - format: date-time - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - type: string - maxLength: 32768 - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - type: integer - format: int64 - minimum: 0 - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: - description: status of the condition, one of True, False, Unknown. - type: string - enum: - - "True" - - "False" - - Unknown - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - manifestConditions: - description: ManifestConditions represents the conditions of each resource in work deployed on spoke cluster. - type: array - items: - description: ManifestCondition represents the conditions of the resources deployed on spoke cluster - type: object - required: - - conditions - properties: - conditions: - description: Conditions represents the conditions of this resource on spoke cluster - type: array - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - type: object - required: - - lastTransitionTime - - message - - reason - - status - - type - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - type: string - format: date-time - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - type: string - maxLength: 32768 - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - type: integer - format: int64 - minimum: 0 - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: - description: status of the condition, one of True, False, Unknown. - type: string - enum: - - "True" - - "False" - - Unknown - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - identifier: - description: resourceId represents a identity of a resource linking to manifests in spec. - type: object + description: "Condition contains details for one aspect of + the current state of this API Resource. --- This struct + is intended for direct use as an array at the field path + .status.conditions. For example, type FooStatus struct{ + // Represents the observations of a foo's current state. + // Known .status.conditions.type are: \"Available\", \"Progressing\", + and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields + }" properties: - group: - description: Group is the group of the resource. - type: string - kind: - description: Kind is the kind of the resource. + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should + be when the underlying condition changed. If that is + not known, then using the time when the API field changed + is acceptable. + format: date-time type: string - name: - description: Name is the name of the resource + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 type: string - namespace: - description: Namespace is the namespace of the resource, the resource is cluster scoped if the value is empty - type: string - ordinal: - description: Ordinal represents an index in manifests list, so the condition can still be linked to a manifest even thougth manifest cannot be parsed successfully. + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 type: integer - resource: - description: Resource is the resource type of the resource + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last transition. + Producers of specific condition types may define expected + values and meanings for this field, and whether the + values are considered a guaranteed API. The value should + be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown type: string - version: - description: Version is the version of the resource. + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across + resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability + to deconflict is important. The regex it matches is + (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + identifier: + description: resourceId represents a identity of a resource + linking to manifests in spec. + properties: + group: + description: Group is the group of the resource. + type: string + kind: + description: Kind is the kind of the resource. + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: Namespace is the namespace of the resource, + the resource is cluster scoped if the value is empty + type: string + ordinal: + description: Ordinal represents an index in manifests list, + so the condition can still be linked to a manifest even + thougth manifest cannot be parsed successfully. + type: integer + resource: + description: Resource is the resource type of the resource + type: string + version: + description: Version is the version of the resource. + type: string + required: + - ordinal + type: object + required: + - conditions + type: object + type: array + required: + - conditions + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 63cb2aa16..0002303d1 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -55,7 +55,7 @@ func AddToManager(m manager.Manager) error { return nil } -// CreateValidatingWebhookConfiguration creates the validatingwebhookconfiguration object for the webhook +// CreateFleetWebhookConfiguration creates the ValidatingWebhookConfiguration object for the webhook func CreateFleetWebhookConfiguration(ctx context.Context, client client.Client, caPEM []byte, port int) error { failPolicy := admv1.Fail // reject request if the webhook doesn't work sideEffortsNone := admv1.SideEffectClassNone diff --git a/test/e2e/framework/namespace.go b/test/e2e/framework/namespace.go index be1984ede..3e582c36a 100644 --- a/test/e2e/framework/namespace.go +++ b/test/e2e/framework/namespace.go @@ -20,7 +20,7 @@ import ( func CreateNamespace(cluster Cluster, ns *corev1.Namespace) { ginkgo.By(fmt.Sprintf("Creating Namespace(%s)", ns.Name), func() { err := cluster.KubeClient.Create(context.TODO(), ns) - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(err).Should(gomega.Succeed()) }) }