Refactor GPU Operator Support

This reduces the Kubernetes API interface down to "please install autoscaler/GPU or not". The actual logic is delegated to the controller proper that's able to make the intelligent descision based on presence of a workload pool that requires the feature, and specifics of that flavor. While in the area, I've removed the CAPO specific hacks from the K8S API and pushed them down into the `clusteropenstack` provisioner where they belong, likewise scheduling hints for the autoscaler.
unikorn-cloud · Sep 25, 2024 · 43677ca · 43677ca
1 parent 8901985
commit 43677ca
Show file tree

Hide file tree

Showing 15 changed files with 307 additions and 344 deletions.
diff --git a/charts/kubernetes/Chart.yaml b/charts/kubernetes/Chart.yaml
@@ -4,8 +4,8 @@ description: A Helm chart for deploying Unikorn Kubernetes Service
 
 type: application
 
-version: v0.2.37
-appVersion: v0.2.37
+version: v0.2.38
+appVersion: v0.2.38
 
 icon: https://raw.githubusercontent.com/unikorn-cloud/assets/main/images/logos/dark-on-light/icon.png
 

diff --git a/charts/kubernetes/crds/unikorn-cloud.org_clustermanagerapplicationbundles.yaml b/charts/kubernetes/crds/unikorn-cloud.org_clustermanagerapplicationbundles.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.16.1
+    controller-gen.kubebuilder.io/version: v0.16.3
   name: clustermanagerapplicationbundles.unikorn-cloud.org
 spec:
   group: unikorn-cloud.org

diff --git a/charts/kubernetes/crds/unikorn-cloud.org_clustermanagers.yaml b/charts/kubernetes/crds/unikorn-cloud.org_clustermanagers.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.16.1
+    controller-gen.kubebuilder.io/version: v0.16.3
   name: clustermanagers.unikorn-cloud.org
 spec:
   group: unikorn-cloud.org

diff --git a/charts/kubernetes/crds/unikorn-cloud.org_kubernetesclusterapplicationbundles.yaml b/charts/kubernetes/crds/unikorn-cloud.org_kubernetesclusterapplicationbundles.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.16.1
+    controller-gen.kubebuilder.io/version: v0.16.3
   name: kubernetesclusterapplicationbundles.unikorn-cloud.org
 spec:
   group: unikorn-cloud.org

diff --git a/charts/kubernetes/crds/unikorn-cloud.org_kubernetesclusters.yaml b/charts/kubernetes/crds/unikorn-cloud.org_kubernetesclusters.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.16.1
+    controller-gen.kubebuilder.io/version: v0.16.3
   name: kubernetesclusters.unikorn-cloud.org
 spec:
   group: unikorn-cloud.org
@@ -278,18 +278,14 @@ spec:
                     description: |-
                       DiskSize is the persistent root disk size to deploy with.  This
                       overrides the default ephemeral disk size defined in the flavor.
+                      This is irrelevant for baremetal machine flavors.
                     pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                     x-kubernetes-int-or-string: true
                   flavorId:
-                    description: Flavor is the OpenStack Nova flavor to deploy with.
-                    type: string
-                  flavorName:
-                    description: |-
-                      FlavorName is the name of the flavor.
-                      CAPO is broken and doesn't accept an ID, so we need to use this.
+                    description: Flavor is the regions service flavor to deploy with.
                     type: string
                   imageId:
-                    description: Image is the OpenStack Glance image to deploy with.
+                    description: Image is the region service image to deploy with.
                     type: string
                   replicas:
                     default: 3
@@ -298,7 +294,6 @@ spec:
                     type: integer
                 required:
                 - flavorId
-                - flavorName
                 - imageId
                 type: object
               features:
@@ -307,31 +302,32 @@ spec:
                 properties:
                   autoscaling:
                     description: |-
-                      Autoscaling, if true, provisions a cluster autoscaler
-                      and allows workload pools to specify autoscaling configuration.
+                      Autoscaling enables the provision of a cluster autoscaler.
+                      This is only installed if a workload pool has autoscaling enabled.
                     type: boolean
-                  nvidiaOperator:
+                  gpuOperator:
                     description: |-
-                      NvidiaOperator, if false do not install the Nvidia Operator, otherwise
-                      install if GPU flavors are detected
+                      GPUOperator enables the provision of a GPU operator.
+                      This is only installed if a workload pool has a flavor that defines
+                      a valid GPU specification and vendor.
                     type: boolean
                 type: object
               network:
                 description: Network defines the Kubernetes networking.
                 properties:
                   dnsNameservers:
-                    description: |-
-                      DNSNameservers sets the DNS nameservers for pods.
-                      At present due to some technical challenges, this must contain
-                      only one DNS server.
+                    description: DNSNameservers sets the DNS nameservers for hosts
+                      on the network.
                     items:
                       pattern: ^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])$
                       type: string
                     minItems: 1
                     type: array
                     x-kubernetes-list-type: set
                   nodeNetwork:
-                    description: NodeNetwork is the IPv4 prefix for the node network.
+                    description: |-
+                      NodeNetwork is the IPv4 prefix for the node network.
+                      This is tyically used to populate a physical network address range.
                     pattern: ^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\/(?:3[0-2]|[1-2]?[0-9])$
                     type: string
                   podNetwork:
@@ -389,50 +385,6 @@ spec:
                                 this pool can be scaled down to.
                               minimum: 0
                               type: integer
-                            scheduler:
-                              description: |-
-                                Scheduler is required when scale-from-zero support is requested
-                                i.e. MimumumReplicas is 0.  This provides scheduling hints to
-                                the autoscaler as it cannot derive CPU/memory constraints from
-                                the machine flavor.
-                              properties:
-                                cpu:
-                                  description: CPU defines the number of CPUs for
-                                    the pool flavor.
-                                  minimum: 1
-                                  type: integer
-                                gpu:
-                                  description: |-
-                                    GPU needs to be set when the pool contains GPU resources so
-                                    the autoscaler can make informed choices when scaling up.
-                                  properties:
-                                    count:
-                                      description: Count is the number of GPUs for
-                                        the pool flavor.
-                                      minimum: 1
-                                      type: integer
-                                    type:
-                                      description: Type is the type of GPU.
-                                      enum:
-                                      - nvidia.com/gpu
-                                      type: string
-                                  required:
-                                  - count
-                                  - type
-                                  type: object
-                                memory:
-                                  anyOf:
-                                  - type: integer
-                                  - type: string
-                                  description: |-
-                                    Memory defines the amount of memory for the pool flavor.
-                                    Internally this will be rounded down to the nearest Gi.
-                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                                  x-kubernetes-int-or-string: true
-                              required:
-                              - cpu
-                              - memory
-                              type: object
                           required:
                           - maximumReplicas
                           - minimumReplicas
@@ -447,6 +399,7 @@ spec:
                           description: |-
                             DiskSize is the persistent root disk size to deploy with.  This
                             overrides the default ephemeral disk size defined in the flavor.
+                            This is irrelevant for baremetal machine flavors.
                           pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                           x-kubernetes-int-or-string: true
                         files:
@@ -471,16 +424,11 @@ spec:
                             type: object
                           type: array
                         flavorId:
-                          description: Flavor is the OpenStack Nova flavor to deploy
+                          description: Flavor is the regions service flavor to deploy
                             with.
                           type: string
-                        flavorName:
-                          description: |-
-                            FlavorName is the name of the flavor.
-                            CAPO is broken and doesn't accept an ID, so we need to use this.
-                          type: string
                         imageId:
-                          description: Image is the OpenStack Glance image to deploy
+                          description: Image is the region service image to deploy
                             with.
                           type: string
                         labels:
@@ -500,7 +448,6 @@ spec:
                           type: integer
                       required:
                       - flavorId
-                      - flavorName
                       - imageId
                       - name
                       type: object

diff --git a/charts/kubernetes/templates/applications.yaml b/charts/kubernetes/templates/applications.yaml
@@ -303,6 +303,32 @@ spec:
 ---
 apiVersion: unikorn-cloud.org/v1alpha1
 kind: HelmApplication
+metadata:
+  name: {{ include "resource.id" "amd-gpu-operator" }}
+  labels:
+    {{- include "unikorn.labels" . | nindent 4 }}
+    unikorn-cloud.org/name: amd-gpu-operator
+  annotations:
+    unikorn-cloud.org/description: |-
+      Provides AMD GPU support in Kubernetes clusters.
+spec:
+  documentation: https://github.com/ROCm/gpu-operator
+  license: Apache-2.0 License
+  icon: todo
+  tags:
+  - infrastructure
+  versions:
+  - version: 0.13.0
+    repo: https://rocm.github.io/k8s-device-plugin
+    chart: amd-gpu
+    parameters:
+    - name: nfd.enabled
+      value: 'true'
+    - name: labeller.enabled
+      value: 'true'
+---
+apiVersion: unikorn-cloud.org/v1alpha1
+kind: HelmApplication
 metadata:
   name: {{ include "resource.id" "cluster-autoscaler" }}
   labels:

diff --git a/charts/kubernetes/templates/kubernetesclusterapplicationbundles.yaml b/charts/kubernetes/templates/kubernetesclusterapplicationbundles.yaml
@@ -37,6 +37,11 @@ spec:
       kind: HelmApplication
       name: {{ include "resource.id" "nvidia-gpu-operator" }}
       version: v23.9.1
+  - name: amd-gpu-operator
+    reference:
+      kind: HelmApplication
+      name: {{ include "resource.id" "amd-gpu-operator" }}
+      version: 0.13.0
   - name: cluster-autoscaler
     reference:
       kind: HelmApplication

diff --git a/pkg/apis/unikorn/v1alpha1/helpers.go b/pkg/apis/unikorn/v1alpha1/helpers.go
@@ -142,9 +142,9 @@ func (c *KubernetesCluster) AutoscalingEnabled() bool {
 	return c.Spec.Features != nil && c.Spec.Features.Autoscaling != nil && *c.Spec.Features.Autoscaling
 }
 
-// NvidiaOperatorEnabled indicates whether to install the Nvidia GPU operator.
-func (c *KubernetesCluster) NvidiaOperatorEnabled() bool {
-	return c.Spec.Features != nil && c.Spec.Features.NvidiaOperator != nil && *c.Spec.Features.NvidiaOperator
+// GPUOperatorEnabled indicates whether to install the GPU operator.
+func (c *KubernetesCluster) GPUOperatorEnabled() bool {
+	return c.Spec.Features != nil && c.Spec.Features.GPUOperator != nil && *c.Spec.Features.GPUOperator
 }
 
 func CompareClusterManager(a, b ClusterManager) int {

diff --git a/pkg/apis/unikorn/v1alpha1/types.go b/pkg/apis/unikorn/v1alpha1/types.go
@@ -20,7 +20,6 @@ package v1alpha1
 import (
 	unikornv1core "github.com/unikorn-cloud/core/pkg/apis/unikorn/v1alpha1"
 
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -71,15 +70,6 @@ type ClusterManagerStatus struct {
 	Conditions []unikornv1core.Condition `json:"conditions,omitempty"`
 }
 
-// MachineGeneric contains common things across all pool types, including
-// Kubernetes cluster manager nodes and workload pools.
-type MachineGeneric struct {
-	unikornv1core.MachineGeneric `json:",inline"`
-	// FlavorName is the name of the flavor.
-	// CAPO is broken and doesn't accept an ID, so we need to use this.
-	FlavorName *string `json:"flavorName"`
-}
-
 // File is a file that can be deployed to a cluster node on creation.
 type File struct {
 	// Path is the absolute path to create the file in.
@@ -99,42 +89,12 @@ type MachineGenericAutoscaling struct {
 	// this pool can be scaled up to.
 	// +kubebuilder:validation:Minimum=1
 	MaximumReplicas *int `json:"maximumReplicas"`
-	// Scheduler is required when scale-from-zero support is requested
-	// i.e. MimumumReplicas is 0.  This provides scheduling hints to
-	// the autoscaler as it cannot derive CPU/memory constraints from
-	// the machine flavor.
-	Scheduler *MachineGenericAutoscalingScheduler `json:"scheduler,omitempty"`
-}
-
-// MachineGenericAutoscalingScheduler defines generic autoscaling scheduling
-// constraints.
-type MachineGenericAutoscalingScheduler struct {
-	// CPU defines the number of CPUs for the pool flavor.
-	// +kubebuilder:validation:Minimum=1
-	CPU *int `json:"cpu"`
-	// Memory defines the amount of memory for the pool flavor.
-	// Internally this will be rounded down to the nearest Gi.
-	Memory *resource.Quantity `json:"memory"`
-	// GPU needs to be set when the pool contains GPU resources so
-	// the autoscaler can make informed choices when scaling up.
-	GPU *MachineGenericAutoscalingSchedulerGPU `json:"gpu,omitempty"`
-}
-
-// MachineGenericAutoscalingSchedulerGPU defines generic autoscaling
-// scheduling constraints for GPUs.
-type MachineGenericAutoscalingSchedulerGPU struct {
-	// Type is the type of GPU.
-	// +kubebuilder:validation:Enum=nvidia.com/gpu
-	Type *string `json:"type"`
-	// Count is the number of GPUs for the pool flavor.
-	// +kubebuilder:validation:Minimum=1
-	Count *int `json:"count"`
 }
 
 // KubernetesWorkloadPoolSpec defines the requested machine pool
 // state.
 type KubernetesWorkloadPoolSpec struct {
-	MachineGeneric `json:",inline"`
+	unikornv1core.MachineGeneric `json:",inline"`
 	// Name is the name of the pool.
 	Name string `json:"name"`
 	// Labels is the set of node labels to apply to the pool on
@@ -193,7 +153,7 @@ type KubernetesClusterSpec struct {
 	// API defines Kubernetes API specific options.
 	API *KubernetesClusterAPISpec `json:"api,omitempty"`
 	// ControlPlane defines the cluster manager topology.
-	ControlPlane *KubernetesClusterControlPlaneSpec `json:"controlPlane"`
+	ControlPlane *unikornv1core.MachineGeneric `json:"controlPlane"`
 	// WorkloadPools defines the workload cluster topology.
 	WorkloadPools *KubernetesClusterWorkloadPoolsSpec `json:"workloadPools"`
 	// Features defines add-on features that can be enabled for the cluster.
@@ -227,16 +187,13 @@ type KubernetesClusterNetworkSpec struct {
 }
 
 type KubernetesClusterFeaturesSpec struct {
-	// Autoscaling, if true, provisions a cluster autoscaler
-	// and allows workload pools to specify autoscaling configuration.
+	// Autoscaling enables the provision of a cluster autoscaler.
+	// This is only installed if a workload pool has autoscaling enabled.
 	Autoscaling *bool `json:"autoscaling,omitempty"`
-	// NvidiaOperator, if false do not install the Nvidia Operator, otherwise
-	// install if GPU flavors are detected
-	NvidiaOperator *bool `json:"nvidiaOperator,omitempty"`
-}
-
-type KubernetesClusterControlPlaneSpec struct {
-	MachineGeneric `json:",inline"`
+	// GPUOperator enables the provision of a GPU operator.
+	// This is only installed if a workload pool has a flavor that defines
+	// a valid GPU specification and vendor.
+	GPUOperator *bool `json:"gpuOperator,omitempty"`
 }
 
 type KubernetesClusterWorkloadPoolsPoolSpec struct {