diff --git a/Dockerfile b/Dockerfile index 62223169d..ebe5c9d66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ ########################################################## # Build the manager binary -FROM golang:1.20 as builder +FROM golang:1.22.1 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/Makefile b/Makefile index 5ab0d27b8..a19aedf37 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 1.21.5 +VERSION ?= latest # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -159,7 +159,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi CONTROLLER_GEN = $(shell pwd)/bin/controller-gen .PHONY: controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@latest) KUSTOMIZE = $(shell pwd)/bin/kustomize .PHONY: kustomize diff --git a/api/v1beta1/slice_types.go b/api/v1beta1/slice_types.go index f4ba95acc..d45c9f6b8 100644 --- a/api/v1beta1/slice_types.go +++ b/api/v1beta1/slice_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( + controllerv1alpha1 "github.com/kubeslice/apis/pkg/controller/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -56,13 +57,13 @@ type SliceConfig struct { // display name of the slice. SliceDisplayName string `json:"sliceDisplayName"` // IP subnet range of the slice. - SliceSubnet string `json:"sliceSubnet"` + SliceSubnet string `json:"sliceSubnet,omitempty"` // Type of the slice. SliceType string `json:"sliceType"` // QOS profile details - QosProfileDetails QosProfileDetails `json:"qosProfileDetails"` + QosProfileDetails QosProfileDetails `json:"qosProfileDetails,omitempty"` // IPAM configuration for the slice - SliceIpam SliceIpamConfig `json:"sliceIpam"` + SliceIpam SliceIpamConfig `json:"sliceIpam,omitempty"` // ExternalGatewayConfig determines istio ingress/egress configuration ExternalGatewayConfig *ExternalGatewayConfig `json:"externalGatewayConfig,omitempty"` // Namespace Isolation profile contains fields related to namespace binding to slice @@ -73,8 +74,8 @@ type SliceConfig struct { SliceGatewayServiceType string `json:"sliceGatewayServiceType,omitempty"` // SliceGateway Protocol Type: UDP or TCP SliceGatewayProtocol string `json:"sliceGatewayProtocol,omitempty"` - // Slice overlay network deployment mode: single-network or multi-network - SliceOverlayNetworkDeploymentMode string `json:"sliceOverlayNetworkDeploymentMode,omitempty"` + // Slice overlay network deployment mode: single-network, multi-network or no-network + SliceOverlayNetworkDeploymentMode controllerv1alpha1.NetworkType `json:"sliceOverlayNetworkDeploymentMode,omitempty"` } // NamespaceIsolationProfile defines the namespace isolation policy for the slice diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 5dc1af2fe..59d12142b 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* Copyright 2022. diff --git a/config/crd/bases/networking.kubeslice.io_serviceexports.yaml b/config/crd/bases/networking.kubeslice.io_serviceexports.yaml index bce42f0d9..98220b823 100644 --- a/config/crd/bases/networking.kubeslice.io_serviceexports.yaml +++ b/config/crd/bases/networking.kubeslice.io_serviceexports.yaml @@ -1,11 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: serviceexports.networking.kubeslice.io spec: group: networking.kubeslice.io @@ -46,14 +44,19 @@ spec: description: ServiceExport is the Schema for the serviceexports API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -61,8 +64,9 @@ spec: description: ServiceExportSpec defines the desired state of ServiceExport properties: aliases: - description: Alias names for the exported service. The service could - be addressed by the alias names in addition to the slice.local name. + description: |- + Alias names for the exported service. The service could be addressed by the alias names + in addition to the slice.local name. items: type: string type: array @@ -84,8 +88,9 @@ spec: type: string protocol: default: TCP - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults - to "TCP". + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string servicePort: description: Port number of the exported service @@ -114,24 +119,24 @@ spec: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -144,13 +149,13 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic slice: description: Slice denotes the slice which the app is part of type: string @@ -163,8 +168,9 @@ spec: description: ServiceExportStatus defines the observed state of ServiceExport properties: aliases: - description: Alias names for the exported service. The service could - be addressed by the alias names in addition to the slice.local name. + description: |- + Alias names for the exported service. The service could be addressed by the alias names + in addition to the slice.local name. items: type: string type: array @@ -178,8 +184,9 @@ spec: description: ExportStatus denotes the export status of the service type: string exposedPorts: - description: ExposedPorts shows a one line representation of ports - and protocols exposed only used to show as a printercolumn + description: |- + ExposedPorts shows a one line representation of ports and protocols exposed + only used to show as a printercolumn type: string ingressGwEnabled: description: IngressGwEnabled denotes ingress gw is enabled for the @@ -235,9 +242,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/bases/networking.kubeslice.io_serviceimports.yaml b/config/crd/bases/networking.kubeslice.io_serviceimports.yaml index 7428bdb11..239114d0d 100644 --- a/config/crd/bases/networking.kubeslice.io_serviceimports.yaml +++ b/config/crd/bases/networking.kubeslice.io_serviceimports.yaml @@ -1,11 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: serviceimports.networking.kubeslice.io spec: group: networking.kubeslice.io @@ -40,14 +38,19 @@ spec: description: ServiceImport is the Schema for the serviceimports API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -55,8 +58,9 @@ spec: description: ServiceImportSpec defines the desired state of ServiceImport properties: aliases: - description: Alias names for the exported service. The service could - be addressed by the alias names in addition to the slice.local name. + description: |- + Alias names for the exported service. The service could be addressed by the alias names + in addition to the slice.local name. items: type: string type: array @@ -77,8 +81,9 @@ spec: type: string protocol: default: TCP - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults - to "TCP". + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". type: string servicePort: description: Port number of the exported service @@ -143,8 +148,9 @@ spec: type: object type: array exposedPorts: - description: ExposedPorts shows a one line representation of ports - and protocols exposed only used to show as a printercolumn + description: |- + ExposedPorts shows a one line representation of ports and protocols exposed + only used to show as a printercolumn type: string importStatus: description: ImportStatus denotes the status of the imported service @@ -163,9 +169,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/bases/networking.kubeslice.io_slicegateways.yaml b/config/crd/bases/networking.kubeslice.io_slicegateways.yaml index f53d78477..614d1c509 100644 --- a/config/crd/bases/networking.kubeslice.io_slicegateways.yaml +++ b/config/crd/bases/networking.kubeslice.io_slicegateways.yaml @@ -1,11 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: slicegateways.networking.kubeslice.io spec: group: networking.kubeslice.io @@ -38,14 +36,19 @@ spec: description: SliceGateway is the Schema for the slicegateways API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -223,9 +226,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/bases/networking.kubeslice.io_slices.yaml b/config/crd/bases/networking.kubeslice.io_slices.yaml index e6d64c135..3b7d18992 100644 --- a/config/crd/bases/networking.kubeslice.io_slices.yaml +++ b/config/crd/bases/networking.kubeslice.io_slices.yaml @@ -1,11 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: slices.networking.kubeslice.io spec: group: networking.kubeslice.io @@ -22,14 +20,19 @@ spec: description: Slice is the Schema for the slices API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -184,8 +187,12 @@ spec: - sliceIpamType type: object sliceOverlayNetworkDeploymentMode: - description: 'Slice overlay network deployment mode: single-network - or multi-network' + description: 'Slice overlay network deployment mode: single-network, + multi-network or no-network' + enum: + - single-network + - multi-network + - no-network type: string sliceSubnet: description: IP subnet range of the slice. @@ -194,11 +201,8 @@ spec: description: Type of the slice. type: string required: - - qosProfileDetails - sliceDisplayName - sliceId - - sliceIpam - - sliceSubnet - sliceType type: object type: object @@ -207,9 +211,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e998cd581..d4baf12e7 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,9 +1,7 @@ - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: manager-role rules: - apiGroups: diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index c2941a191..ee691def5 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -1,9 +1,7 @@ - --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - creationTimestamp: null name: mutating-webhook-configuration webhooks: - admissionReviewVersions: diff --git a/controllers/controller.go b/controllers/controller.go index 3c50826f4..c1d249f02 100644 --- a/controllers/controller.go +++ b/controllers/controller.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/go-logr/logr" + "github.com/kubeslice/apis/pkg/controller/v1alpha1" kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" "github.com/kubeslice/worker-operator/pkg/logger" corev1 "k8s.io/api/core/v1" @@ -186,7 +187,7 @@ func SliceAppNamespaceConfigured(ctx context.Context, slice string, namespace st return false, nil } -func GetSliceOverlayNetworkType(ctx context.Context, c client.Client, sliceName string) (string, error) { +func GetSliceOverlayNetworkType(ctx context.Context, c client.Client, sliceName string) (v1alpha1.NetworkType, error) { slice, err := GetSlice(ctx, c, sliceName) if err != nil { return "", err diff --git a/controllers/slice/net_op.go b/controllers/slice/net_op.go index 52603369c..a186878a1 100644 --- a/controllers/slice/net_op.go +++ b/controllers/slice/net_op.go @@ -25,6 +25,7 @@ import ( "github.com/kubeslice/worker-operator/pkg/logger" "github.com/kubeslice/worker-operator/pkg/netop" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -97,6 +98,10 @@ func (r *SliceReconciler) SendSliceDeletionEventToNetOp(ctx context.Context, sli // This populates the NetOpPods map in the slice reconciler structure. err := r.getNetOpPods(ctx, sliceName, namespace) if err != nil { + if errors.IsNotFound(err) { + //early exit since there are no object found + return nil + } return err } diff --git a/controllers/slice/reconciler.go b/controllers/slice/reconciler.go index 3b572987c..e6a6e40ec 100644 --- a/controllers/slice/reconciler.go +++ b/controllers/slice/reconciler.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/kubeslice/apis/pkg/controller/v1alpha1" "github.com/kubeslice/kubeslice-monitoring/pkg/events" "github.com/kubeslice/kubeslice-monitoring/pkg/metrics" kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" @@ -129,6 +130,7 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrl.Result{}, err } } + // Examine DeletionTimestamp to determine if object is under deletion // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent @@ -161,56 +163,19 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return res, err } - debugLog.Info("Syncing slice QoS config with NetOp pods") - err = r.SyncSliceQosProfileWithNetOp(ctx, slice) - if err != nil { - log.Error(err, "Failed to sync QoS profile with netop pods") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceQoSProfileWithNetOpsSync, controllerName) + if slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode == v1alpha1.NONET { + debugLog.Info("No communication slice, skipping reconciliation of qos, netop, egw, router etc") + // to support net to no-net switching write a function to delete network components if present } else { - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceUpdated, controllerName) - } - - log.Info("ExternalGatewayConfig", "egw", slice.Status.SliceConfig) - - if isEgressConfigured(slice) { - debugLog.Info("Installing egress") - err = manifest.InstallEgress(ctx, r.Client, slice) - if err != nil { - log.Error(err, "unable to install egress") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceEgressInstallFailed, controllerName) - return ctrl.Result{}, nil - } - } - - if isIngressConfigured(slice) { - debugLog.Info("Installing ingress") - err = manifest.InstallIngress(ctx, r.Client, slice) - if err != nil { - log.Error(err, "unable to install ingress") - utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceIngressInstallFailed, controllerName) - return ctrl.Result{}, nil + debugLog.Info("Slice with network, continue reconciliation of qos, netop, egw, router etc") + // syncQoStoNetop, reconcile slice router, slicegw edge, ext gateways + res, err, requeue := r.reconcileNetworkComponents(ctx, slice) + if requeue { + debugLog.Info("Retry reconciling SliceNetworkComponents", "res", res, "err", err) + return res, err } } - res, err, requeue = r.ReconcileSliceRouter(ctx, slice) - if err != nil { - log.Error(err, "Failed to reconcile slice router") - } - if requeue { - return res, err - } - - res, err, requeue = r.ReconcileSliceGwEdge(ctx, slice) - if err != nil { - log.Error(err, "Slice Edge reconciliation failed") - return res, err - } - if requeue { - return ctrl.Result{ - Requeue: true, - }, nil - } - appPods, err := r.getAppPods(ctx, slice) debugLog.Info("app pods", "pods", appPods, "err", err) @@ -247,6 +212,63 @@ func (r *SliceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl }, nil } +func (r *SliceReconciler) reconcileNetworkComponents(ctx context.Context, slice *kubeslicev1beta1.Slice) (reconcile.Result, error, bool) { + log := logger.FromContext(ctx) + debugLog := log.V(1) + + debugLog.Info("Syncing slice QoS config with NetOp pods") + err := r.SyncSliceQosProfileWithNetOp(ctx, slice) + if err != nil { + log.Error(err, "Failed to sync QoS profile with netop pods") + utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceQoSProfileWithNetOpsSync, controllerName) + } else { + utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceUpdated, controllerName) + } + + debugLog.Info("reconciling SliceRouter") + res, err, requeue := r.ReconcileSliceRouter(ctx, slice) + if err != nil { + log.Error(err, "Failed to reconcile slice router") + return res, err, true + } + if requeue { + return res, nil, true + } + + debugLog.Info("reconciling SliceGwEdge") + res, err, requeue = r.ReconcileSliceGwEdge(ctx, slice) + if err != nil { + log.Error(err, "Slice Edge reconciliation failed") + return res, err, true + } + if requeue { + return res, nil, true + } + + debugLog.Info("ExternalGatewayConfig", "egw", slice.Status.SliceConfig) + if isEgressConfigured(slice) { + debugLog.Info("Installing egress") + err = manifest.InstallEgress(ctx, r.Client, slice) + if err != nil { + log.Error(err, "unable to install egress") + utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceEgressInstallFailed, controllerName) + return ctrl.Result{}, err, false + } + } + + if isIngressConfigured(slice) { + debugLog.Info("Installing ingress") + err = manifest.InstallIngress(ctx, r.Client, slice) + if err != nil { + log.Error(err, "unable to install ingress") + utils.RecordEvent(ctx, r.EventRecorder, slice, nil, ossEvents.EventSliceIngressInstallFailed, controllerName) + return ctrl.Result{}, err, false + } + } + + return ctrl.Result{}, nil, false +} + func (r *SliceReconciler) exposeMetric(appPods []kubeslicev1beta1.AppPod, slice *kubeslicev1beta1.Slice) { // this extra check is needed when current app pods are zero and slice status has old app pods // then it doesn't goes to app pods loop hence it don't update the app pods metrics count to zero @@ -300,10 +322,9 @@ func (r *SliceReconciler) handleDnsSvc(ctx context.Context, slice *kubeslicev1be if err != nil { if errors.IsNotFound(err) { - log.Error(err, "dns not found") - log.Info("DNS service not found in the cluster, probably coredns is not deployed; continuing") + debugLog.Info("DNS service not found in the cluster, probably coredns is not deployed for no-net slice; continuing") } else { - log.Error(err, "Unable to find DNS Service") + log.Error(err, "Failed to get DNS Service") return true, ctrl.Result{}, err } } else { diff --git a/controllers/slice/slicerouter.go b/controllers/slice/slicerouter.go index 5efce8d4b..5db2d8b96 100644 --- a/controllers/slice/slicerouter.go +++ b/controllers/slice/slicerouter.go @@ -460,7 +460,7 @@ func newDeploymentSliceRouter(r *SliceReconciler, ctx context.Context, slice *ku func sliceConfigDefined(slice *kubeslicev1beta1.Slice) bool { return slice.Status.SliceConfig != nil && slice.Status.SliceConfig.SliceSubnet != "" && slice.Status.SliceConfig.ClusterSubnetCIDR != "" } -func (r *SliceReconciler) cleanupSliceRouter(ctx context.Context, sliceName string) error { +func (r *SliceReconciler) cleanupVl3NSE(ctx context.Context, sliceName string) error { log := logger.FromContext(ctx) vl3Nse := &nsmv1.NetworkService{} diff --git a/controllers/slice/utils.go b/controllers/slice/utils.go index 3d0339551..22ab6e770 100644 --- a/controllers/slice/utils.go +++ b/controllers/slice/utils.go @@ -20,6 +20,7 @@ package slice import ( "context" + kubeslicev1beta1 "github.com/kubeslice/worker-operator/api/v1beta1" "github.com/kubeslice/worker-operator/controllers" corev1 "k8s.io/api/core/v1" @@ -35,7 +36,7 @@ func (r *SliceReconciler) cleanupSliceResources(ctx context.Context, slice *kube return err } //cleanup slice router network service - if err := r.cleanupSliceRouter(ctx, slice.Name); err != nil { + if err := r.cleanupVl3NSE(ctx, slice.Name); err != nil { return err } //cleanup Service Discovery objects - serviceimport and export objects that belong to this slice diff --git a/controllers/slicegateway/reconciler.go b/controllers/slicegateway/reconciler.go index 48afa26b3..817688c72 100644 --- a/controllers/slicegateway/reconciler.go +++ b/controllers/slicegateway/reconciler.go @@ -23,6 +23,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/kubeslice/apis/pkg/controller/v1alpha1" "github.com/kubeslice/kubeslice-monitoring/pkg/events" ossEvents "github.com/kubeslice/worker-operator/events" "github.com/kubeslice/worker-operator/pkg/utils" @@ -129,6 +130,12 @@ func (r *SliceGwReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct }, nil } + if slice.Status.SliceConfig != nil && + slice.Status.SliceConfig.SliceOverlayNetworkDeploymentMode == v1alpha1.NONET { + log.Info("No communication slice. Skipping slicegw reconcilation") + return ctrl.Result{}, err + } + // Check if slice router network service endpoint (NSE) is present before spawning slice gateway pod. // Gateways connect to vL3 slice router at startup, hence it is necessary to check if the // NSE present before creating the gateway pods. diff --git a/go.mod b/go.mod index 33f089934..26de77582 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kubeslice/worker-operator -go 1.21 +go 1.22.1 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.1 @@ -9,7 +9,7 @@ require ( github.com/go-logr/zapr v1.2.4 //github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.6.0 - github.com/kubeslice/apis v0.2.2 + github.com/kubeslice/apis v0.3.0 github.com/kubeslice/gateway-sidecar v0.2.0 github.com/kubeslice/kubeslice-monitoring v0.1.10 github.com/kubeslice/netops v0.1.3 diff --git a/go.sum b/go.sum index 1b0ec3599..cd620fb12 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeslice/apis v0.2.2 h1:UAxAhEZ+76545qth5Br2cGr9lnj3aV0YbJptoGf61hk= -github.com/kubeslice/apis v0.2.2/go.mod h1:YDSfpIsQM+FtQPaZVGNCTZnlp3viWuQhkjJjIHQdaYs= +github.com/kubeslice/apis v0.3.0 h1:sSaMLKWN9OfUdcTYAcIaUpRwlZqrmgKLPG5I8dYYH2k= +github.com/kubeslice/apis v0.3.0/go.mod h1:YDSfpIsQM+FtQPaZVGNCTZnlp3viWuQhkjJjIHQdaYs= github.com/kubeslice/gateway-sidecar v0.2.0 h1:Ja3fIUivuSjUFQ4lPCt79ATq99BxslvAFYUwV9Urpy4= github.com/kubeslice/gateway-sidecar v0.2.0/go.mod h1:nM1+Wjud2vk44cUg+9iwBbWTpqI+2Ecbn9NuaHEs9aY= github.com/kubeslice/kubeslice-monitoring v0.1.10 h1:ozzGuSxr5dsouI2ATtPOLygsKfQ8w6wUI9QHkSp33TQ= diff --git a/pkg/hub/controllers/cluster/cluster_controller_test.go b/pkg/hub/controllers/cluster/cluster_controller_test.go index 5c24864ea..67a623653 100644 --- a/pkg/hub/controllers/cluster/cluster_controller_test.go +++ b/pkg/hub/controllers/cluster/cluster_controller_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,7 +19,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +/* +FIXME(reduce code repetition): instead of repeating setup/cleanup steps(i.e installing node, nsm configmap, project ns, nsmgr daemonset etc) +for each Context blocks, create a common section for setup & clean up that executes before the actual validations/tests. +*/ var _ = Describe("Hub ClusterController", func() { + var nsmgrMock *appsv1.DaemonSet Context("With Cluster CR Created at hub cluster", func() { var ns *corev1.Namespace var cluster *hubv1alpha1.Cluster @@ -65,6 +71,32 @@ var _ = Describe("Hub ClusterController", func() { Spec: hubv1alpha1.ClusterSpec{}, Status: hubv1alpha1.ClusterStatus{}, } + // nsmgr daemonset (isNetworkPresent) + nsmgrMock = &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nsmgr", + Namespace: CONTROL_PLANE_NS, + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "nsmgr"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nsmgr", + Image: "containerImage:tag", + }, + }, + }, + }, + }, + } nsmconfig = configMap("nsm-config", "kubeslice-system", ` Prefixes: - 192.168.0.0/16 @@ -86,6 +118,8 @@ var _ = Describe("Hub ClusterController", func() { return k8sClient.Update(ctx, cluster) }) Expect(err).To(BeNil()) + // delete nsmgr + Expect(k8sClient.Delete(ctx, nsmgrMock)).Should(Succeed()) Eventually(func() bool { err := k8sClient.Get(ctx, types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cluster) return errors.IsNotFound(err) @@ -100,6 +134,7 @@ var _ = Describe("Hub ClusterController", func() { if errors.IsNotFound(err) { Expect(k8sClient.Create(ctx, ns)).Should(Succeed()) } + Expect(k8sClient.Create(ctx, nsmgrMock)).Should(Succeed()) Expect(k8sClient.Create(ctx, cluster)).Should(Succeed()) err = k8sClient.Get(ctx, types.NamespacedName{Name: nsmconfig.Name, Namespace: nsmconfig.Namespace}, nsmconfig) @@ -182,10 +217,37 @@ var _ = Describe("Hub ClusterController", func() { os.Setenv("CLUSTER_NAME", cluster.Name) operatorSecret = getSecret("kubeslice-kubernetes-dashboard", CONTROL_PLANE_NS) sa = getSA("kubeslice-kubernetes-dashboard", CONTROL_PLANE_NS, operatorSecret.Name) + // nsmgr daemonset (isNetworkPresent) + nsmgrMock = &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nsmgr", + Namespace: CONTROL_PLANE_NS, + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "nsmgr"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nsmgr", + Image: "containerImage:tag", + }, + }, + }, + }, + }, + } }) It("should create secret in controller's project namespace", func() { os.Setenv("CLUSTER_ENDPOINT", hostname) Expect(k8sClient.Create(ctx, node)) + Expect(k8sClient.Create(ctx, nsmgrMock)).Should(Succeed()) Expect(k8sClient.Create(ctx, cluster)).Should(Succeed()) Expect(k8sClient.Create(ctx, operatorSecret)).Should(Succeed()) Expect(k8sClient.Create(ctx, sa)).Should(Succeed()) @@ -234,6 +296,8 @@ var _ = Describe("Hub ClusterController", func() { var operatorSecret *corev1.Secret var sa *corev1.ServiceAccount var pods []*corev1.Pod + var node *corev1.Node + var nsmconfig *corev1.ConfigMap // var podBasedComponents []string BeforeEach(func() { @@ -242,6 +306,32 @@ var _ = Describe("Hub ClusterController", func() { Name: PROJECT_NS, }, } + node = &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "topology.kubernetes.io/region": "us-east-1", + "kubeslice.io/node-type": "gateway", + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: "gce://demo", + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeExternalIP, + Address: "35.235.10.1", + }, + }, + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } nsSpire = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "spire", @@ -260,6 +350,36 @@ var _ = Describe("Hub ClusterController", func() { Spec: hubv1alpha1.ClusterSpec{}, Status: hubv1alpha1.ClusterStatus{}, } + // nsmgr daemonset (isNetworkPresent) + nsmgrMock = &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nsmgr", + Namespace: CONTROL_PLANE_NS, + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "nsmgr"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nsmgr", + Image: "containerImage:tag", + }, + }, + }, + }, + }, + } + nsmconfig = configMap("nsm-config", "kubeslice-system", ` + Prefixes: + - 192.168.0.0/16 + - 10.96.0.0/12`) // podBasedComponents = []string{ // "nsmgr", // "forwarder", @@ -277,14 +397,24 @@ var _ = Describe("Hub ClusterController", func() { os.Setenv("CLUSTER_NAME", CLUSTER_NAME) operatorSecret = getSecret("kubeslice-kubernetes-dashboard", CONTROL_PLANE_NS) sa = getSA("kubeslice-kubernetes-dashboard", CONTROL_PLANE_NS, operatorSecret.Name) + Expect(k8sClient.Create(ctx, node)) Expect(k8sClient.Create(ctx, ns)) Expect(k8sClient.Create(ctx, nsSpire)) Expect(k8sClient.Create(ctx, nsIstio)) Expect(k8sClient.Create(ctx, operatorSecret)) Expect(k8sClient.Create(ctx, sa)) + err := k8sClient.Get(ctx, types.NamespacedName{Name: nsmconfig.Name, Namespace: nsmconfig.Namespace}, nsmconfig) + if errors.IsNotFound(err) { + Expect(k8sClient.Create(ctx, nsmconfig)).Should(Succeed()) + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: "nsmgr", Namespace: CONTROL_PLANE_NS}, nsmgrMock) + if errors.IsNotFound(err) { + Expect(k8sClient.Create(ctx, nsmgrMock)).Should(Succeed()) + } Expect(k8sClient.Create(ctx, cluster)) DeferCleanup(func() { + Expect(k8sClient.Delete(ctx, node)).Should(Succeed()) // Delete cluster object k8sClient.Delete(ctx, cluster) err := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -305,6 +435,8 @@ var _ = Describe("Hub ClusterController", func() { fmt.Println("----------------- clusterobj finalizer", cluster.ObjectMeta.Finalizers) return err != nil && errors.IsNotFound(err) }, time.Second*100, time.Second*1).Should(BeTrue()) + // delete nsmgr + Expect(k8sClient.Delete(ctx, nsmgrMock)).Should(Succeed()) k8sClient.Delete(ctx, operatorSecret) k8sClient.Delete(ctx, sa) diff --git a/pkg/hub/controllers/cluster/crds/hub.kubeslice.io_clusters.yaml b/pkg/hub/controllers/cluster/crds/controller.kubeslice.io_clusters.yaml similarity index 89% rename from pkg/hub/controllers/cluster/crds/hub.kubeslice.io_clusters.yaml rename to pkg/hub/controllers/cluster/crds/controller.kubeslice.io_clusters.yaml index 044bd5ea5..99886028f 100644 --- a/pkg/hub/controllers/cluster/crds/hub.kubeslice.io_clusters.yaml +++ b/pkg/hub/controllers/cluster/crds/controller.kubeslice.io_clusters.yaml @@ -163,6 +163,11 @@ spec: type: string type: object type: array + networkPresent: + default: false + description: NetworkPresent denotes if the networking components (NSM, + Spire) are installed on a cluster + type: boolean nodeIPs: description: NodeIPs of the gateway node of worker cluster items: @@ -182,15 +187,23 @@ spec: secretName: description: SecretName is the name of the secret for the worker cluster. type: string + vCPURestriction: + description: VCPURestriction is the restriction on the cluster disabling + the creation of new pods + properties: + enforceRestrictions: + description: EnforceRestrictions is the flag to check if the cluster + is restricted + type: boolean + lastUpdatedTimestamp: + description: LastUpdatedTimestamp is the timestamp when the enforcement + was updated + format: date-time + type: string + type: object type: object type: object served: true storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/pkg/hub/controllers/cluster/deregister.go b/pkg/hub/controllers/cluster/deregister.go index 0dc63a0b8..3e44afb2a 100644 --- a/pkg/hub/controllers/cluster/deregister.go +++ b/pkg/hub/controllers/cluster/deregister.go @@ -182,20 +182,8 @@ func (r *Reconciler) updateRegistrationStatus(ctx context.Context, cluster *hubv if err != nil { return err } - if status == hubv1alpha1.RegistrationStatusRegistered { - // TODO: update this status declaration when health check is implemented - // for CNI subnet and Node IP address. - if (cluster.Status.RegistrationStatus == "" || cluster.Status.RegistrationStatus == hubv1alpha1.RegistrationStatusPending) && - len(cluster.Status.CniSubnet) > 0 { - // we mark registered only when status is empty or in pending state and cluster has CNI Subnet - cluster.Status.RegistrationStatus = status - return r.Status().Update(ctx, cluster) - } - return nil - } else { - cluster.Status.RegistrationStatus = status - return r.Status().Update(ctx, cluster) - } + cluster.Status.RegistrationStatus = status + return r.Status().Update(ctx, cluster) }) if err != nil { return err diff --git a/pkg/hub/controllers/cluster/reconciler.go b/pkg/hub/controllers/cluster/reconciler.go index 110df977f..1ddb19f14 100644 --- a/pkg/hub/controllers/cluster/reconciler.go +++ b/pkg/hub/controllers/cluster/reconciler.go @@ -34,8 +34,9 @@ import ( "github.com/kubeslice/worker-operator/pkg/logger" "github.com/kubeslice/worker-operator/pkg/utils" "github.com/prometheus/client_golang/prometheus" - corev1 "k8s.io/api/core/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -89,11 +90,15 @@ var clusterDeregisterFinalizer = "worker.kubeslice.io/cluster-deregister-finaliz func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { log := logger.FromContext(ctx).WithName("cluster-reconciler") + debuglog := log.V(1) ctx = logger.WithLogger(ctx, log) cr, err := r.getCluster(ctx, req) if cr == nil { return reconcile.Result{}, err } + if cr.Status.RegistrationStatus == "" { + cr.Status.RegistrationStatus = hubv1alpha1.RegistrationStatusInProgress + } log.Info("got cluster CR from hub", "cluster", cr) requeue, result, err := r.handleClusterDeletion(cr, ctx, req) if requeue { @@ -110,6 +115,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco return res, err } utils.RecordEvent(ctx, r.EventRecorder, cr, nil, ossEvents.EventClusterProviderUpdateInfoSuccesful, controllerName) + debuglog.Info("cluster status", "cr.Status.NetworkPresent", cr.Status.NetworkPresent) + if err = r.updateNetworkStatus(ctx, cr); err != nil { + log.Error(err, "unable to update networkPresent status") + } res, err, requeue = r.updateCNISubnetConfig(ctx, cr, cl) if err != nil { utils.RecordEvent(ctx, r.EventRecorder, cr, nil, ossEvents.EventClusterCNISubnetUpdateFailed, controllerName) @@ -118,7 +127,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco if requeue { return res, err } - // Update registration status to registered + // Update registration status to registered when slice networking enabled & cluster staus has CNI Subnet list + // if cluster.Status.NetworkPresent + if cr.Status.NetworkPresent && len(cr.Status.CniSubnet) == 0 { + debuglog.Info("registration status: pending, as cni subnet list not populated") + if err = r.updateRegistrationStatus(ctx, cr, hubv1alpha1.RegistrationStatusPending); err != nil { + log.Error(err, "unable to update registration status") + } + return reconcile.Result{Requeue: true}, nil + } + + debuglog.Info("Update registration status to registered") if err = r.updateRegistrationStatus(ctx, cr, hubv1alpha1.RegistrationStatusRegistered); err != nil { log.Error(err, "unable to update registration status") } @@ -169,36 +188,59 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco return reconcile.Result{RequeueAfter: r.ReconcileInterval}, nil } +// if nsm core compoent is not found in a cluster, it's assumed that +// the kubeslice network dependencies have not been installed in it +func (r *Reconciler) isNsmInstalled(ctx context.Context) bool { + log := logger.FromContext(ctx) + debuglog := log.V(1) + dsList := &appsv1.DaemonSetList{} + listOpts := []client.ListOption{ + client.MatchingLabels(map[string]string{"app": "nsmgr"}), + client.InNamespace(ControlPlaneNamespace), + } + if err := r.MeshClient.List(ctx, dsList, listOpts...); err != nil { + log.Error(err, "Failed to list nsmgr ds") + return false + } + debuglog.Info("isNsmInstalled", "nsmgr ds count", len(dsList.Items)) + return len(dsList.Items) != 0 +} + func (r *Reconciler) updateClusterHealthStatus(ctx context.Context, cr *hubv1alpha1.Cluster) error { log := logger.FromContext(ctx) + debuglog := log.V(1) csList := []hubv1alpha1.ComponentStatus{} var chs hubv1alpha1.ClusterHealthStatus = hubv1alpha1.ClusterHealthStatusNormal - for _, c := range components { - cs, err := r.getComponentStatus(ctx, &c, cr) - if err != nil { - log.Error(err, "unable to fetch component status") - } - if cs != nil { - csList = append(csList, *cs) - if cs.ComponentHealthStatus != hubv1alpha1.ComponentHealthStatusNormal { - chs = hubv1alpha1.ClusterHealthStatusWarning - log.Info("Component unhealthy", "component", c.name) + if cr.Status.NetworkPresent { + debuglog.Info("Worker installation with network component, continue health check") + + for _, c := range components { + cs, err := r.getComponentStatus(ctx, &c, cr) + if err != nil { + log.Error(err, "unable to fetch component status") + } + if cs != nil { + csList = append(csList, *cs) + if cs.ComponentHealthStatus != hubv1alpha1.ComponentHealthStatusNormal { + chs = hubv1alpha1.ClusterHealthStatusWarning + debuglog.Info("Component unhealthy", "component", c.name) + } } } - } - // Cluster health changed, raise event - if cr.Status.ClusterHealth.ClusterHealthStatus != chs { - if chs == hubv1alpha1.ClusterHealthStatusNormal { - log.Info("cluster health is back to normal") - utils.RecordEvent(ctx, r.EventRecorder, cr, nil, ossEvents.EventClusterHealthy, controllerName) - } else if chs == hubv1alpha1.ClusterHealthStatusWarning { - log.Info("cluster health is in warning state") - utils.RecordEvent(ctx, r.EventRecorder, cr, nil, ossEvents.EventClusterUnhealthy, controllerName) + // Cluster health changed, raise event + if cr.Status.ClusterHealth.ClusterHealthStatus != chs { + if chs == hubv1alpha1.ClusterHealthStatusNormal { + log.Info("cluster health is back to normal") + utils.RecordEvent(ctx, r.EventRecorder, cr, nil, ossEvents.EventClusterHealthy, controllerName) + } else if chs == hubv1alpha1.ClusterHealthStatusWarning { + log.Info("cluster health is in warning state") + utils.RecordEvent(ctx, r.EventRecorder, cr, nil, ossEvents.EventClusterUnhealthy, controllerName) + } } - } + } cr.Status.ClusterHealth.ComponentStatuses = csList cr.Status.ClusterHealth.ClusterHealthStatus = chs @@ -237,6 +279,7 @@ func (r *Reconciler) getComponentStatus(ctx context.Context, c *component, cr *h } return cs, nil } + if c.name == "cni-subnets" { if cr == nil { return nil, nil @@ -295,6 +338,27 @@ func (r *Reconciler) getComponentStatus(ctx context.Context, c *component, cr *h return cs, nil } +func (r *Reconciler) updateNetworkStatus(ctx context.Context, cluster *hubv1alpha1.Cluster) error { + log := logger.FromContext(ctx) + debuglog := log.V(1) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := r.Get(ctx, types.NamespacedName{ + Name: cluster.Name, + Namespace: cluster.Namespace, + }, cluster) + if err != nil { + return err + } + cluster.Status.NetworkPresent = r.isNsmInstalled(ctx) + debuglog.Info("checked nsm deploy", "isNsmInstalled", cluster.Status.NetworkPresent) + return r.Status().Update(ctx, cluster) + }) + if err != nil { + return err + } + return nil +} + func (r *Reconciler) updateClusterCloudProviderInfo(ctx context.Context, cr *hubv1alpha1.Cluster, cl cluster.ClusterInterface) (ctrl.Result, error, bool) { log := logger.FromContext(ctx) clusterInfo, err := cl.GetClusterInfo(ctx) @@ -338,6 +402,7 @@ func (r *Reconciler) updateClusterCloudProviderInfo(ctx context.Context, cr *hub func (r *Reconciler) updateCNISubnetConfig(ctx context.Context, cr *hubv1alpha1.Cluster, cl cluster.ClusterInterface) (ctrl.Result, error, bool) { log := logger.FromContext(ctx) + debuglog := log.V(1) toUpdate := false err := retry.RetryOnConflict(retry.DefaultRetry, func() error { err := r.Get(ctx, types.NamespacedName{ @@ -347,16 +412,18 @@ func (r *Reconciler) updateCNISubnetConfig(ctx context.Context, cr *hubv1alpha1. if err != nil { return err } - cniSubnet, err := cl.GetNsmExcludedPrefix(ctx, "nsm-config", "kubeslice-system") - if err != nil { - log.Error(err, "Failed to get nsm config") - return err + cniSubnet := []string{} + // nsm-config is present only when operator is installed with networking enabled + if cr.Status.NetworkPresent { + debuglog.Info("try to get cni subnets from nsm-config cm") + cniSubnet, err = cl.GetNsmExcludedPrefix(ctx, "nsm-config", "kubeslice-system") + if err != nil { + log.Error(err, "Failed to get nsm config") + return err + } } if !reflect.DeepEqual(cr.Status.CniSubnet, cniSubnet) { cr.Status.CniSubnet = cniSubnet - // TODO: move this status declaration to outside this function when health check is implemented - // for CNI subnet and Node IP address. - cr.Status.RegistrationStatus = hubv1alpha1.RegistrationStatusRegistered toUpdate = true return r.Status().Update(ctx, cr) } diff --git a/pkg/hub/controllers/slice_controller.go b/pkg/hub/controllers/slice_controller.go index 05fcdc538..2176983c7 100644 --- a/pkg/hub/controllers/slice_controller.go +++ b/pkg/hub/controllers/slice_controller.go @@ -24,6 +24,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/kubeslice/apis/pkg/controller/v1alpha1" spokev1alpha1 "github.com/kubeslice/apis/pkg/worker/v1alpha1" "github.com/kubeslice/kubeslice-monitoring/pkg/events" "github.com/kubeslice/kubeslice-monitoring/pkg/metrics" @@ -386,6 +387,11 @@ func (r *SliceReconciler) updateSliceHealth(ctx context.Context, slice *spokev1a debuglog := log.V(1) slice.Status.SliceHealth.ComponentStatuses = []spokev1alpha1.ComponentStatus{} slice.Status.SliceHealth.SliceHealthStatus = spokev1alpha1.SliceHealthStatusNormal + if slice.Spec.OverlayNetworkDeploymentMode == v1alpha1.NONET { + debuglog.Info("skipping workerslice health check for no-network deployment") + return nil + } + // check for components since slice network is enabled for _, c := range components { cs, err := r.getComponentStatus(ctx, &c, slice.Spec.SliceName) if err != nil { diff --git a/pkg/webhook/pod/webhook.go b/pkg/webhook/pod/webhook.go index 0606abca5..df997a154 100644 --- a/pkg/webhook/pod/webhook.go +++ b/pkg/webhook/pod/webhook.go @@ -24,6 +24,7 @@ import ( "fmt" "net/http" + "github.com/kubeslice/apis/pkg/controller/v1alpha1" "github.com/kubeslice/worker-operator/controllers" "github.com/kubeslice/worker-operator/pkg/logger" v1 "k8s.io/api/admission/v1" @@ -51,7 +52,7 @@ var ( type SliceInfoProvider interface { SliceAppNamespaceConfigured(ctx context.Context, slice string, namespace string) (bool, error) GetNamespaceLabels(ctx context.Context, client client.Client, namespace string) (map[string]string, error) - GetSliceOverlayNetworkType(ctx context.Context, client client.Client, sliceName string) (string, error) + GetSliceOverlayNetworkType(ctx context.Context, client client.Client, sliceName string) (v1alpha1.NetworkType, error) } type WebhookServer struct { @@ -301,7 +302,7 @@ func (wh *WebhookServer) MutationRequired(metadata metav1.ObjectMeta, ctx contex log.Error(err, "Error getting slice overlay network type") return false, "" } - if sliceNetworkType != "" && sliceNetworkType != "single-network" { + if sliceNetworkType != "" && sliceNetworkType != v1alpha1.SINGLENET { log.Info("Slice overlay type is not single-network. Skip pod mutation...") return false, "" } diff --git a/pkg/webhook/pod/webhook_test.go b/pkg/webhook/pod/webhook_test.go index 081de286a..0ea9332e7 100644 --- a/pkg/webhook/pod/webhook_test.go +++ b/pkg/webhook/pod/webhook_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/kubeslice/apis/pkg/controller/v1alpha1" "github.com/kubeslice/worker-operator/controllers" "github.com/kubeslice/worker-operator/pkg/webhook/pod" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,7 +45,7 @@ func (f fakeWebhookClient) GetNamespaceLabels(ctx context.Context, client client return map[string]string{controllers.ApplicationNamespaceSelectorLabelKey: "green"}, nil } -func (f fakeWebhookClient) GetSliceOverlayNetworkType(ctx context.Context, client client.Client, sliceName string) (string, error) { +func (f fakeWebhookClient) GetSliceOverlayNetworkType(ctx context.Context, client client.Client, sliceName string) (v1alpha1.NetworkType, error) { return "", nil } diff --git a/pkg/webhook/pod/webhook_utils.go b/pkg/webhook/pod/webhook_utils.go index 42b9938ad..4589350e9 100644 --- a/pkg/webhook/pod/webhook_utils.go +++ b/pkg/webhook/pod/webhook_utils.go @@ -3,6 +3,7 @@ package pod import ( "context" + "github.com/kubeslice/apis/pkg/controller/v1alpha1" "github.com/kubeslice/worker-operator/controllers" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -32,6 +33,6 @@ func (w *webhookClient) GetNamespaceLabels(ctx context.Context, client client.Cl return nsLabels, nil } -func (w *webhookClient) GetSliceOverlayNetworkType(ctx context.Context, client client.Client, sliceName string) (string, error) { +func (w *webhookClient) GetSliceOverlayNetworkType(ctx context.Context, client client.Client, sliceName string) (v1alpha1.NetworkType, error) { return controllers.GetSliceOverlayNetworkType(ctx, client, sliceName) } diff --git a/test.Dockerfile b/test.Dockerfile index ea3819758..c1c6bd15a 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20 as builder +FROM golang:1.22.1 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/tests/files/crd/hub.kubeslice.io_clusters.yaml b/tests/files/crd/controller.kubeslice.io_clusters.yaml similarity index 89% rename from tests/files/crd/hub.kubeslice.io_clusters.yaml rename to tests/files/crd/controller.kubeslice.io_clusters.yaml index 044bd5ea5..99886028f 100644 --- a/tests/files/crd/hub.kubeslice.io_clusters.yaml +++ b/tests/files/crd/controller.kubeslice.io_clusters.yaml @@ -163,6 +163,11 @@ spec: type: string type: object type: array + networkPresent: + default: false + description: NetworkPresent denotes if the networking components (NSM, + Spire) are installed on a cluster + type: boolean nodeIPs: description: NodeIPs of the gateway node of worker cluster items: @@ -182,15 +187,23 @@ spec: secretName: description: SecretName is the name of the secret for the worker cluster. type: string + vCPURestriction: + description: VCPURestriction is the restriction on the cluster disabling + the creation of new pods + properties: + enforceRestrictions: + description: EnforceRestrictions is the flag to check if the cluster + is restricted + type: boolean + lastUpdatedTimestamp: + description: LastUpdatedTimestamp is the timestamp when the enforcement + was updated + format: date-time + type: string + type: object type: object type: object served: true storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/tests/files/crd/hub.kubeslice.io_projects.yaml b/tests/files/crd/controller.kubeslice.io_projects.yaml similarity index 93% rename from tests/files/crd/hub.kubeslice.io_projects.yaml rename to tests/files/crd/controller.kubeslice.io_projects.yaml index 671101313..999ca6564 100644 --- a/tests/files/crd/hub.kubeslice.io_projects.yaml +++ b/tests/files/crd/controller.kubeslice.io_projects.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: projects.controller.kubeslice.io spec: @@ -58,9 +57,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/tests/files/crd/hub.kubeslice.io_serviceexportconfigs.yaml b/tests/files/crd/controller.kubeslice.io_serviceexportconfigs.yaml similarity index 84% rename from tests/files/crd/hub.kubeslice.io_serviceexportconfigs.yaml rename to tests/files/crd/controller.kubeslice.io_serviceexportconfigs.yaml index 7d5cf9bdc..0957ab058 100644 --- a/tests/files/crd/hub.kubeslice.io_serviceexportconfigs.yaml +++ b/tests/files/crd/controller.kubeslice.io_serviceexportconfigs.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: serviceexportconfigs.controller.kubeslice.io spec: @@ -37,6 +36,12 @@ spec: spec: description: ServiceExportConfigSpec defines the desired state of ServiceExportConfig properties: + aliases: + description: Alias names for the exported service. The service could + be addressed by the alias names in addition to the slice.local name. + items: + type: string + type: array serviceDiscoveryEndpoints: description: the service discovery endpoint array items: @@ -67,12 +72,19 @@ spec: description: The name of the port. type: string port: - description: The port number. + description: The target port number. format: int32 type: integer protocol: description: The protocol. type: string + servicePort: + description: The port exposed by service + format: int32 + type: integer + serviceProtocol: + description: The service protocol + type: string type: object type: array serviceName: @@ -99,9 +111,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/tests/files/crd/hub.kubeslice.io_sliceconfigs.yaml b/tests/files/crd/controller.kubeslice.io_sliceconfigs.yaml similarity index 63% rename from tests/files/crd/hub.kubeslice.io_sliceconfigs.yaml rename to tests/files/crd/controller.kubeslice.io_sliceconfigs.yaml index 7889a3603..897026162 100644 --- a/tests/files/crd/hub.kubeslice.io_sliceconfigs.yaml +++ b/tests/files/crd/controller.kubeslice.io_sliceconfigs.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: sliceconfigs.controller.kubeslice.io spec: @@ -104,6 +103,13 @@ spec: default: false type: boolean type: object + overlayNetworkDeploymentMode: + default: single-network + enum: + - single-network + - multi-network + - no-network + type: string qosProfileDetails: description: The custom QOS Profile Details properties: @@ -144,6 +150,15 @@ spec: - queueType - tcType type: object + renewBefore: + description: RenewBefore is used for renew now! + format: date-time + type: string + rotationInterval: + default: 30 + maximum: 90 + minimum: 30 + type: integer sliceGatewayProvider: description: WorkerSliceGatewayProvider defines the configuration for slicegateway @@ -151,6 +166,29 @@ spec: sliceCaType: default: Local type: string + sliceGatewayServiceType: + items: + properties: + cluster: + type: string + protocol: + default: UDP + enum: + - TCP + - UDP + type: string + type: + default: NodePort + enum: + - NodePort + - LoadBalancer + type: string + required: + - cluster + - protocol + - type + type: object + type: array sliceGatewayType: default: OpenVPN type: string @@ -168,20 +206,67 @@ spec: type: string standardQosProfileName: type: string + vpnConfig: + description: VPNConfiguration defines the additional (optional) VPN + Configuration to customise + properties: + cipher: + default: AES-256-CBC + enum: + - AES-256-CBC + - AES-128-CBC + type: string + required: + - cipher + type: object required: - - sliceGatewayProvider + - maxClusters type: object status: description: SliceConfigStatus defines the observed state of SliceConfig + properties: + kubesliceEvents: + items: + properties: + action: + description: Trigger action. Examples - CLUSTER_OFFBOARDING, + NAMESPCE_OFFBOARDING etc + type: string + components: + description: list of effected components on which action failed + items: + type: string + type: array + event: + description: Event name (from monitoring framework schema) + type: string + identifier: + description: Identifier of the component for which the action + was triggered + type: string + isEventRaised: + default: false + description: Flag to determine if kubernetes event is already + raised + type: boolean + reason: + description: Reason message for the event + type: string + timestamp: + description: Timestamp of the event + format: date-time + type: string + type: + description: Type of the event. Can be one of Error, Success + or InProgress + type: string + required: + - event + type: object + type: array type: object type: object served: true storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/tests/files/crd/networking.kubeslice.io_slices.yaml b/tests/files/crd/networking.kubeslice.io_slices.yaml index 732454b4b..6ac65aeb4 100644 --- a/tests/files/crd/networking.kubeslice.io_slices.yaml +++ b/tests/files/crd/networking.kubeslice.io_slices.yaml @@ -162,6 +162,12 @@ spec: sliceDisplayName: description: display name of the slice. type: string + sliceGatewayProtocol: + description: 'SliceGateway Protocol Type: UDP or TCP' + type: string + sliceGatewayServiceType: + description: SliceGateway Service Type + type: string sliceId: description: UUID of the slice. type: string @@ -177,6 +183,14 @@ spec: required: - sliceIpamType type: object + sliceOverlayNetworkDeploymentMode: + description: 'Slice overlay network deployment mode: single-network, + multi-network or no-network' + enum: + - single-network + - multi-network + - no-network + type: string sliceSubnet: description: IP subnet range of the slice. type: string @@ -184,11 +198,8 @@ spec: description: Type of the slice. type: string required: - - qosProfileDetails - sliceDisplayName - sliceId - - sliceIpam - - sliceSubnet - sliceType type: object type: object diff --git a/tests/files/crd/spoke.kubeslice.io_spokeserviceimports.yaml b/tests/files/crd/worker.kubeslice.io_workerserviceimports.yaml similarity index 84% rename from tests/files/crd/spoke.kubeslice.io_spokeserviceimports.yaml rename to tests/files/crd/worker.kubeslice.io_workerserviceimports.yaml index dc2be6742..038c43090 100644 --- a/tests/files/crd/spoke.kubeslice.io_spokeserviceimports.yaml +++ b/tests/files/crd/worker.kubeslice.io_workerserviceimports.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: workerserviceimports.worker.kubeslice.io spec: @@ -37,6 +36,12 @@ spec: spec: description: WorkerServiceImportSpec defines the desired state of WorkerServiceImport properties: + aliases: + description: Alias names for the exported service. The service could + be addressed by the alias names in addition to the slice.local name. + items: + type: string + type: array serviceDiscoveryEndpoints: description: the service discovery endpoint array items: @@ -67,12 +72,19 @@ spec: description: The name of the port. type: string port: - description: The port number. + description: The target port number. format: int32 type: integer protocol: description: The protocol. type: string + servicePort: + description: The port exposed by service + format: int32 + type: integer + serviceProtocol: + description: The service protocol + type: string type: object type: array serviceName: @@ -99,9 +111,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/tests/files/crd/spoke.kubeslice.io_spokesliceconfigs.yaml b/tests/files/crd/worker.kubeslice.io_workersliceconfigs.yaml similarity index 91% rename from tests/files/crd/spoke.kubeslice.io_spokesliceconfigs.yaml rename to tests/files/crd/worker.kubeslice.io_workersliceconfigs.yaml index 51a8303b0..c0cafd960 100644 --- a/tests/files/crd/spoke.kubeslice.io_spokesliceconfigs.yaml +++ b/tests/files/crd/worker.kubeslice.io_workersliceconfigs.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: workersliceconfigs.worker.kubeslice.io spec: @@ -77,6 +76,15 @@ spec: default: false type: boolean type: object + octet: + type: integer + overlayNetworkDeploymentMode: + default: single-network + enum: + - single-network + - multi-network + - no-network + type: string qosProfileDetails: description: QOSProfile is the QOS Profile configuration from backend properties: @@ -116,6 +124,18 @@ spec: sliceCaType: default: Local type: string + sliceGatewayProtocol: + default: UDP + enum: + - TCP + - UDP + type: string + sliceGatewayServiceType: + default: NodePort + enum: + - NodePort + - LoadBalancer + type: string sliceGatewayType: default: OpenVPN type: string @@ -130,8 +150,6 @@ spec: sliceType: default: Application type: string - required: - - ipamClusterOctet type: object status: description: WorkerSliceConfigStatus defines the observed state of Slice @@ -208,9 +226,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/tests/files/crd/spoke.kubeslice.io_spokeslicegateways.yaml b/tests/files/crd/worker.kubeslice.io_workerslicegateways.yaml similarity index 85% rename from tests/files/crd/spoke.kubeslice.io_spokeslicegateways.yaml rename to tests/files/crd/worker.kubeslice.io_workerslicegateways.yaml index c3f80ab9b..8266a588d 100644 --- a/tests/files/crd/spoke.kubeslice.io_spokeslicegateways.yaml +++ b/tests/files/crd/worker.kubeslice.io_workerslicegateways.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.1 creationTimestamp: null name: workerslicegateways.worker.kubeslice.io spec: @@ -35,6 +35,12 @@ spec: spec: description: WorkerSliceGatewaySpec defines the desired state of WorkerSliceGateway properties: + gatewayConnectivityType: + default: NodePort + enum: + - NodePort + - LoadBalancer + type: string gatewayCredentials: properties: secretName: @@ -47,6 +53,12 @@ spec: type: string gatewayNumber: type: integer + gatewayProtocol: + default: UDP + enum: + - TCP + - UDP + type: string gatewayType: default: OpenVPN type: string @@ -58,6 +70,10 @@ spec: type: string gatewaySubnet: type: string + loadBalancerIps: + items: + type: string + type: array nodeIp: type: string nodeIps: @@ -81,6 +97,10 @@ spec: type: string gatewaySubnet: type: string + loadBalancerIps: + items: + type: string + type: array nodeIp: type: string nodeIps: @@ -112,9 +132,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/tests/hub/node_restart_test.go b/tests/hub/node_restart_test.go index fd74bf7d0..2905831c8 100644 --- a/tests/hub/node_restart_test.go +++ b/tests/hub/node_restart_test.go @@ -26,6 +26,7 @@ import ( hubv1alpha1 "github.com/kubeslice/apis/pkg/controller/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,6 +39,7 @@ var _ = Describe("NodeRestart Test Suite", func() { var ns *v1.Namespace var cluster *hubv1alpha1.Cluster var nsmconfig *v1.ConfigMap + var nsmgrMock *appsv1.DaemonSet Context("With kubeslice node restarting", func() { BeforeEach(func() { @@ -158,6 +160,33 @@ var _ = Describe("NodeRestart Test Suite", func() { if errors.IsNotFound(err) { Expect(k8sClient.Create(ctx, ns)).Should(Succeed()) } + // nsmgr daemonset (isNetworkPresent) + nsmgrMock = &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nsmgr", + Namespace: CONTROL_PLANE_NS, + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "nsmgr"}, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "nsmgr"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nsmgr", + Image: "containerImage:tag", + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, nsmgrMock)).Should(Succeed()) // create cluster CR under project namespace cluster = &hubv1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{ @@ -204,6 +233,8 @@ Prefixes: return k8sClient.Update(ctx, cluster) }) Expect(err).To(BeNil()) + // delete nsmgr + Expect(k8sClient.Delete(ctx, nsmgrMock)).Should(Succeed()) Eventually(func() bool { err := k8sClient.Delete(ctx, node1) return errors.IsNotFound(err) diff --git a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/cluster_types.go b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/cluster_types.go index d29a54855..63efbe842 100644 --- a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/cluster_types.go +++ b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/cluster_types.go @@ -125,6 +125,9 @@ type ClusterStatus struct { RegistrationStatus RegistrationStatus `json:"registrationStatus,omitempty"` // IsDeregisterInProgress is the flag to check if the cluster deregister is InProgress IsDeregisterInProgress bool `json:"isDeregisterInProgress,omitempty"` + // NetworkPresent denotes if the networking components (NSM, Spire) are installed on a cluster + //+kubebuilder:default:=false + NetworkPresent bool `json:"networkPresent,omitempty"` // VCPURestriction is the restriction on the cluster disabling the creation of new pods VCPURestriction *VCPURestriction `json:"vCPURestriction,omitempty"` diff --git a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go index 28e14439a..7cb391358 100644 --- a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go +++ b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/sliceconfig_types.go @@ -23,16 +23,29 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// +kubebuilder:validation:Enum:=single-network;multi-network;no-network +type NetworkType string + +const ( + // all workloads would be connected to the slice l3 overlay network + SINGLENET NetworkType = "single-network" + + // workloads would be connected at l7 through network of envoy gateways. + // And the gateways would be connected through slice l3 overlay + MULTINET NetworkType = "multi-network" + + // slice without any connectivity between clusters + NONET NetworkType = "no-network" +) + // SliceConfigSpec defines the desired state of SliceConfig type SliceConfigSpec struct { //+kubebuilder:default:=single-network - //+kubebuilder:validation:Enum:=single-network;multi-network - OverlayNetworkDeploymentMode string `json:"overlayNetworkDeploymentMode,omitempty"` - SliceSubnet string `json:"sliceSubnet,omitempty"` + OverlayNetworkDeploymentMode NetworkType `json:"overlayNetworkDeploymentMode,omitempty"` + SliceSubnet string `json:"sliceSubnet,omitempty"` //+kubebuilder:default:=Application - SliceType string `json:"sliceType,omitempty"` - // +kubebuilder:validation:Required - SliceGatewayProvider WorkerSliceGatewayProvider `json:"sliceGatewayProvider"` + SliceType string `json:"sliceType,omitempty"` + SliceGatewayProvider *WorkerSliceGatewayProvider `json:"sliceGatewayProvider,omitempty"` //+kubebuilder:default:=Local SliceIpamType string `json:"sliceIpamType,omitempty"` Clusters []string `json:"clusters,omitempty"` @@ -44,7 +57,7 @@ type SliceConfigSpec struct { //+kubebuilder:validation:Minimum=2 //+kubebuilder:validation:Maximum=32 //+kubebuilder:default:=16 - MaxClusters int `json:"maxClusters,omitempty"` + MaxClusters int `json:"maxClusters"` //+kubebuilder:validation:Minimum=30 //+kubebuilder:validation:Maximum=90 //+kubebuilder:default:=30 diff --git a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/zz_generated.deepcopy.go b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/zz_generated.deepcopy.go index cff368620..0490636be 100644 --- a/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/github.com/kubeslice/apis/pkg/controller/v1alpha1/zz_generated.deepcopy.go @@ -673,7 +673,11 @@ func (in *SliceConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SliceConfigSpec) DeepCopyInto(out *SliceConfigSpec) { *out = *in - in.SliceGatewayProvider.DeepCopyInto(&out.SliceGatewayProvider) + if in.SliceGatewayProvider != nil { + in, out := &in.SliceGatewayProvider, &out.SliceGatewayProvider + *out = new(WorkerSliceGatewayProvider) + (*in).DeepCopyInto(*out) + } if in.Clusters != nil { in, out := &in.Clusters, &out.Clusters *out = make([]string, len(*in)) diff --git a/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go b/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go index 7e6c3684a..6d89ad2ab 100644 --- a/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go +++ b/vendor/github.com/kubeslice/apis/pkg/worker/v1alpha1/workersliceconfig_types.go @@ -17,6 +17,7 @@ package v1alpha1 import ( + controllerv1alpha1 "github.com/kubeslice/apis/pkg/controller/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -47,13 +48,11 @@ type WorkerSliceConfigSpec struct { QosProfileDetails QOSProfile `json:"qosProfileDetails,omitempty"` NamespaceIsolationProfile NamespaceIsolationProfile `json:"namespaceIsolationProfile,omitempty"` IpamClusterOctet int `json:"ipamClusterOctet,omitempty"` - //+kubebuilder:validation:Required - Octet *int `json:"octet"` - ClusterSubnetCIDR string `json:"clusterSubnetCIDR,omitempty"` - ExternalGatewayConfig ExternalGatewayConfig `json:"externalGatewayConfig,omitempty"` + Octet *int `json:"octet,omitempty"` + ClusterSubnetCIDR string `json:"clusterSubnetCIDR,omitempty"` + ExternalGatewayConfig ExternalGatewayConfig `json:"externalGatewayConfig,omitempty"` //+kubebuilder:default:=single-network - //+kubebuilder:validation:Enum:=single-network;multi-network - OverlayNetworkDeploymentMode string `json:"overlayNetworkDeploymentMode,omitempty"` + OverlayNetworkDeploymentMode controllerv1alpha1.NetworkType `json:"overlayNetworkDeploymentMode,omitempty"` } // WorkerSliceGatewayProvider defines the configuration for slicegateway diff --git a/vendor/modules.txt b/vendor/modules.txt index b9d14573c..8ca72310d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -115,7 +115,7 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/kubeslice/apis v0.2.2 +# github.com/kubeslice/apis v0.3.0 ## explicit; go 1.16 github.com/kubeslice/apis/pkg/controller/v1alpha1 github.com/kubeslice/apis/pkg/worker/v1alpha1