Skip to content
This repository has been archived by the owner on Apr 11, 2024. It is now read-only.

Commit

Permalink
feat: deploy Nutanix CCM addon
Browse files Browse the repository at this point in the history
Aligns the method of deploying the CCM with all other addons.
  • Loading branch information
dkoshkin committed Apr 8, 2024
1 parent 130d4ec commit ef132d9
Show file tree
Hide file tree
Showing 9 changed files with 367 additions and 8 deletions.
24 changes: 23 additions & 1 deletion api/v1alpha1/addon_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,12 +311,34 @@ func (CSI) VariableSchema() clusterv1.VariableSchema {
}

// CCM tells us to enable or disable the cloud provider interface.
type CCM struct{}
type CCM struct {
// A reference to the Secret for credential information for the target Prism Central instance
// +optional
Credentials *corev1.LocalObjectReference `json:"credentials"`
}

func (CCM) VariableSchema() clusterv1.VariableSchema {
// TODO Validate credentials is set.
// This CCM is shared across all providers.
// Some of these providers may require credentials to be set, but we don't want to require it for all providers.
// The Nutanix CCM handler will fail in at runtime if credentials are not set.
return clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "object",
Properties: map[string]clusterv1.JSONSchemaProps{
"credentials": {
Description: "A reference to the Secret for credential information" +
"for the target Prism Central instance",
Type: "object",
Properties: map[string]clusterv1.JSONSchemaProps{
"name": {
Description: "The name of the Secret",
Type: "string",
},
},
Required: []string{"name"},
},
},
},
}
}
3 changes: 2 additions & 1 deletion api/v1alpha1/clusterconfig_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ const (
CSIProviderAWSEBS = "aws-ebs"
CSIProviderNutanix = "nutanix"

CCMProviderAWS = "aws"
CCMProviderAWS = "aws"
CCMProviderNutanix = "nutanix"
)

// +kubebuilder:object:root=true
Expand Down
7 changes: 6 additions & 1 deletion api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pkg/handlers/generic/lifecycle/ccm/aws/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"

"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client"
lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
Expand Down Expand Up @@ -56,6 +57,7 @@ func New(
func (a *AWSCCM) Apply(
ctx context.Context,
cluster *clusterv1.Cluster,
_ *v1alpha1.ClusterConfigSpec,
) error {
log := ctrl.LoggerFrom(ctx).WithValues(
"cluster",
Expand Down
28 changes: 25 additions & 3 deletions pkg/handlers/generic/lifecycle/ccm/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ const (
)

type CCMProvider interface {
Apply(context.Context, *clusterv1.Cluster) error
Apply(context.Context, *clusterv1.Cluster, *v1alpha1.ClusterConfigSpec) error
}

type CCMHandler struct {
Expand Down Expand Up @@ -78,7 +78,7 @@ func (c *CCMHandler) AfterControlPlaneInitialized(
)
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
resp.SetMessage(
fmt.Sprintf("failed to read CCM provider from cluster definition: %v",
fmt.Sprintf("failed to read CCM from cluster definition: %v",
err,
),
)
Expand All @@ -88,17 +88,39 @@ func (c *CCMHandler) AfterControlPlaneInitialized(
log.V(4).Info("Skipping CCM handler.")
return
}

clusterConfigVar, _, err := variables.Get[v1alpha1.ClusterConfigSpec](
varMap,
clusterconfig.MetaVariableName,
)
if err != nil {
log.Error(
err,
"failed to read clusterConfig variable from cluster definition",
)
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
resp.SetMessage(
fmt.Sprintf("ffailed to read clusterConfig variable from cluster definition: %v",
err,
),
)
return
}

infraKind := req.Cluster.Spec.InfrastructureRef.Kind
log.Info(fmt.Sprintf("finding CCM handler for %s", infraKind))
var handler CCMProvider
switch {
case strings.Contains(strings.ToLower(infraKind), v1alpha1.CCMProviderAWS):
handler = c.ProviderHandler[v1alpha1.CCMProviderAWS]
case strings.Contains(strings.ToLower(infraKind), v1alpha1.CCMProviderNutanix):
handler = c.ProviderHandler[v1alpha1.CCMProviderNutanix]
default:
log.Info(fmt.Sprintf("No CCM handler provided for infra kind %s", infraKind))
return
}
err = handler.Apply(ctx, &req.Cluster)

err = handler.Apply(ctx, &req.Cluster, &clusterConfigVar)
if err != nil {
log.Error(
err,
Expand Down
179 changes: 179 additions & 0 deletions pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
// Copyright 2023 D2iQ, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package nutanix

import (
"bytes"
"context"
"errors"
"fmt"
"text/template"

"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client"
lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils"
"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
)

const (
defaultHelmRepositoryURL = "https://nutanix.github.io/helm/"
defaultHelmChartVersion = "0.3.3"
defaultHelmChartName = "nutanix-cloud-provider"
defaultHelmReleaseName = "nutanix-ccm"
defaultHelmReleaseNamespace = "kube-system"

// This is the name of the Secret on the remote cluster that should match what is defined in Helm values.
//nolint:gosec // Does not contain hard coded credentials.
defaultCredentialsSecretName = "nutanix-ccm-credentials"
)

var ErrMissingCredentials = errors.New("name of the Secret containing PC credentials must be set")

type Config struct {
*options.GlobalOptions

defaultValuesTemplateConfigMapName string
}

func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) {
flags.StringVar(
&c.defaultValuesTemplateConfigMapName,
prefix+".default-values-template-configmap-name",
"default-nutanix-ccm-helm-values-template",
"default values ConfigMap name",
)
}

type provider struct {
client ctrlclient.Client
config *Config
}

func New(
c ctrlclient.Client,
cfg *Config,
) *provider {
return &provider{
client: c,
config: cfg,
}
}

func (p *provider) Apply(
ctx context.Context,
cluster *clusterv1.Cluster,
clusterConfig *v1alpha1.ClusterConfigSpec,
) error {
// No need to check for nil values in the struct, this function will only be called if CCM is not nil
if clusterConfig.Addons.CCM.Credentials == nil {
return ErrMissingCredentials
}

valuesTemplateConfigMap, err := lifecycleutils.RetrieveValuesTemplateConfigMap(
ctx,
p.client,
p.config.defaultValuesTemplateConfigMapName,
p.config.DefaultsNamespace(),
)
if err != nil {
return fmt.Errorf(
"failed to retrieve Nutanix CCM installation values template ConfigMap for cluster: %w",
err,
)
}

// It's possible to have the credentials Secret be created by the Helm chart.
// However, that would leave the credentials visible in the HelmChartProxy.
// Instead, we'll create the Secret on the remote cluster and reference it in the Helm values.
if clusterConfig.Addons.CCM.Credentials != nil {
key := ctrlclient.ObjectKey{
Name: defaultCredentialsSecretName,
Namespace: defaultHelmReleaseNamespace,
}
err = lifecycleutils.CopySecretToRemoteCluster(
ctx,
p.client,
clusterConfig.Addons.CCM.Credentials.Name,
key,
cluster,
)
if err != nil {
return fmt.Errorf(
"error creating Nutanix CCM Credentials Secret on the remote cluster: %w",
err,
)
}
}

values := valuesTemplateConfigMap.Data["values.yaml"]
// The configMap will contain the Helm values, but templated with fields that need to be filled in.
values, err = templateValues(clusterConfig, values)
if err != nil {
return fmt.Errorf("failed to template Helm values read from ConfigMap: %w", err)
}

hcp := &caaphv1.HelmChartProxy{
TypeMeta: metav1.TypeMeta{
APIVersion: caaphv1.GroupVersion.String(),
Kind: "HelmChartProxy",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: cluster.Namespace,
Name: "nutanix-ccm-" + cluster.Name,
},
Spec: caaphv1.HelmChartProxySpec{
RepoURL: defaultHelmRepositoryURL,
ChartName: defaultHelmChartName,
ClusterSelector: metav1.LabelSelector{
MatchLabels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name},
},
ReleaseNamespace: defaultHelmReleaseNamespace,
ReleaseName: defaultHelmReleaseName,
Version: defaultHelmChartVersion,
ValuesTemplate: values,
},
}

if err = controllerutil.SetOwnerReference(cluster, hcp, p.client.Scheme()); err != nil {
return fmt.Errorf(
"failed to set owner reference on nutanix-ccm installation HelmChartProxy: %w",
err,
)
}

if err = client.ServerSideApply(ctx, p.client, hcp); err != nil {
return fmt.Errorf("failed to apply nutanix-ccm installation HelmChartProxy: %w", err)
}

return nil
}

func templateValues(clusterConfig *v1alpha1.ClusterConfigSpec, text string) (string, error) {
helmValuesTemplate, err := template.New("").Parse(text)
if err != nil {
return "", fmt.Errorf("failed to parse Helm values template: %w", err)
}

type input struct {
PrismCentralEndpoint v1alpha1.NutanixPrismCentralEndpointSpec
}
templateInput := input{
PrismCentralEndpoint: clusterConfig.Nutanix.PrismCentralEndpoint,
}

var b bytes.Buffer
err = helmValuesTemplate.Execute(&b, templateInput)
if err != nil {
return "", fmt.Errorf("failed setting PrismCentral configuration in template: %w", err)
}

return b.String(), nil
}
Loading

0 comments on commit ef132d9

Please sign in to comment.