From 3f62b7fc3c0a0ff17452cfb079429b96dfffcf0f Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 24 Jul 2023 10:45:59 +0200 Subject: [PATCH 01/50] Always update elemental-system-agent config --- cmd/register/main.go | 13 ++++++- cmd/register/main_test.go | 42 ++++++++++++++++------- controllers/machineselector_controller.go | 34 ------------------ pkg/install/install.go | 17 +++++++++ pkg/install/mocks/installer.go | 14 ++++++++ 5 files changed, 73 insertions(+), 47 deletions(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index e9a8d9f71..73f4899ba 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -64,6 +64,10 @@ func main() { } func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHandler, installer install.Installer) *cobra.Command { + // Reset config and viper cache + cfg = elementalv1.Config{} + viper.Reset() + // Define command (using closures) cmd := &cobra.Command{ Use: "elemental-register", Short: "Elemental register command", @@ -108,10 +112,17 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa log.Info("Installing Elemental") return installer.InstallElemental(cfg) } + // If the System is already installed, we should update the elemental-system-agent config. + // In case of reset if we just registered to a new MachineInventory. + log.Debug("Updating Elemental System Agent") + if err := installer.UpdateSystemAgentConfig(cfg.Elemental); err != nil { + return fmt.Errorf("updating elemental-system-agent configuration: %w", err) + } + return nil }, } - //Define flags + //Define and bind flags cmd.Flags().StringVar(&cfg.Elemental.Registration.URL, "registration-url", "", "Registration url to get the machine config from") _ = viper.BindPFlag("elemental.registration.url", cmd.Flags().Lookup("registration-url")) cmd.Flags().StringVar(&cfg.Elemental.Registration.CACert, "registration-ca-cert", "", "File with the custom CA certificate to use against he registration url") diff --git a/cmd/register/main_test.go b/cmd/register/main_test.go index 740b723f5..35f522c1f 100644 --- a/cmd/register/main_test.go +++ b/cmd/register/main_test.go @@ -26,7 +26,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" - "github.com/rancher/elemental-operator/pkg/install" imocks "github.com/rancher/elemental-operator/pkg/install/mocks" "github.com/rancher/elemental-operator/pkg/register" rmocks "github.com/rancher/elemental-operator/pkg/register/mocks" @@ -91,15 +90,16 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f var cmd *cobra.Command var mockCtrl *gomock.Controller var client *rmocks.MockClient + var installer *imocks.MockInstaller When("system is already installed", func() { BeforeEach(func() { - fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{ - "/run/initramfs/cos-state/state.yaml": "{}/n", - }) + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) Expect(err).ToNot(HaveOccurred()) mockCtrl = gomock.NewController(GinkgoT()) client = rmocks.NewMockClient(mockCtrl) - cmd = newCommand(fs, client, register.NewFileStateHandler(fs), install.NewInstaller(fs)) + installer = imocks.NewMockInstaller(mockCtrl) + installer.EXPECT().IsSystemInstalled().AnyTimes().Return(true) + cmd = newCommand(fs, client, register.NewFileStateHandler(fs), installer) DeferCleanup(fsCleanup) }) It("should return no error when printing version", func() { @@ -112,7 +112,10 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f }) It("should use the config if no arguments passed", func() { cmd.SetArgs([]string{}) - client.EXPECT().Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)).Return([]byte("{}\n"), nil) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(baseConfigFixture), nil) + installer.EXPECT().UpdateSystemAgentConfig(baseConfigFixture.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should overwrite the config values with passed arguments", func() { @@ -126,14 +129,20 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f }) wantConfig := alternateConfigFixture.DeepCopy() wantConfig.Elemental.Registration.NoSMBIOS = false - client.EXPECT().Register(wantConfig.Elemental.Registration, []byte(wantConfig.Elemental.Registration.CACert)).Return([]byte("{}\n"), nil) + client.EXPECT(). + Register(wantConfig.Elemental.Registration, []byte(wantConfig.Elemental.Registration.CACert)). + Return(marshalToBytes(wantConfig), nil) + installer.EXPECT().UpdateSystemAgentConfig(wantConfig.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should use config path argument", func() { newPath := "/a/custom/config/path/custom-config.yaml" cmd.SetArgs([]string{"--config-path", newPath}) marshalIntoFile(fs, alternateConfigFixture, newPath) - client.EXPECT().Register(alternateConfigFixture.Elemental.Registration, []byte(alternateConfigFixture.Elemental.Registration.CACert)).Return([]byte("{}\n"), nil) + client.EXPECT(). + Register(alternateConfigFixture.Elemental.Registration, []byte(alternateConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(alternateConfigFixture), nil) + installer.EXPECT().UpdateSystemAgentConfig(alternateConfigFixture.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should skip registration if lastUpdate is recent", func() { @@ -153,7 +162,10 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f LastUpdate: time.Now().Add(-25 * time.Hour), } marshalIntoFile(fs, registrationState, defaultStatePath) - client.EXPECT().Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)).Return([]byte("{}\n"), nil) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(baseConfigFixture), nil) + installer.EXPECT().UpdateSystemAgentConfig(baseConfigFixture.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should use state path argument", func() { @@ -176,7 +188,8 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f Expect(err).ToNot(HaveOccurred()) mockCtrl = gomock.NewController(GinkgoT()) installer = imocks.NewMockInstaller(mockCtrl) - installer.EXPECT().IsSystemInstalled().Return(false).AnyTimes() + installer.EXPECT().IsSystemInstalled().AnyTimes().Return(false) + installer.EXPECT().UpdateSystemAgentConfig(gomock.Any()).Times(0) // Only expect update if the system is already installed client = rmocks.NewMockClient(mockCtrl) cmd = newCommand(fs, client, register.NewFileStateHandler(fs), installer) DeferCleanup(fsCleanup) @@ -211,8 +224,13 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f }) func marshalIntoFile(fs vfs.FS, input any, filePath string) { - bytes, err := yaml.Marshal(input) - Expect(err).ToNot(HaveOccurred()) + bytes := marshalToBytes(input) Expect(vfs.MkdirAll(fs, path.Dir(filePath), os.ModePerm)).ToNot(HaveOccurred()) Expect(fs.WriteFile(filePath, bytes, os.ModePerm)).ToNot(HaveOccurred()) } + +func marshalToBytes(input any) []byte { + bytes, err := yaml.Marshal(input) + Expect(err).ToNot(HaveOccurred()) + return bytes +} diff --git a/controllers/machineselector_controller.go b/controllers/machineselector_controller.go index 384dafb89..ee0c241f2 100644 --- a/controllers/machineselector_controller.go +++ b/controllers/machineselector_controller.go @@ -403,25 +403,6 @@ func (r *MachineInventorySelectorReconciler) newBootstrapPlan(ctx context.Contex return "", nil, fmt.Errorf("failed to get a boostrap plan for the machine: %w", err) } - stopAgentPlan := applyinator.Plan{ - OneTimeInstructions: []applyinator.OneTimeInstruction{ - { - CommonInstruction: applyinator.CommonInstruction{ - Command: "systemctl", - Args: []string{ - "stop", - "elemental-system-agent", - }, - }, - }, - }, - } - - stopAgentPlanJSON, err := json.Marshal(stopAgentPlan) - if err != nil { - return "", nil, fmt.Errorf("failed to create new stop elemental agent plan: %w", err) - } - type LabelsFromInventory struct { // NOTE: The '+' is not a typo and is needed when adding labels to k3s/rke // instead of replacing them. @@ -470,11 +451,6 @@ func (r *MachineInventorySelectorReconciler) newBootstrapPlan(ctx context.Contex Path: "/usr/local/etc/hostname", Permissions: "0644", }, - { - Content: base64.StdEncoding.EncodeToString(stopAgentPlanJSON), - Path: "/var/lib/rancher/agent/plans/elemental-agent-stop.plan.skip", - Permissions: "0644", - }, }, OneTimeInstructions: []applyinator.OneTimeInstruction{ { @@ -498,16 +474,6 @@ func (r *MachineInventorySelectorReconciler) newBootstrapPlan(ctx context.Contex }, }, }, - { - // Ensure the local plan can only be executed after bootstrapping script is done - CommonInstruction: applyinator.CommonInstruction{ - Command: "bash", - Args: []string{ - "-c", - "mv /var/lib/rancher/agent/plans/elemental-agent-stop.plan.skip /var/lib/rancher/agent/plans/elemental-agent-stop.plan", - }, - }, - }, }, } diff --git a/pkg/install/install.go b/pkg/install/install.go index a718bc464..6dfa35c77 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -45,6 +45,7 @@ const ( type Installer interface { IsSystemInstalled() bool InstallElemental(config elementalv1.Config) error + UpdateSystemAgentConfig(config elementalv1.Elemental) error } func NewInstaller(fs vfs.FS) Installer { @@ -105,6 +106,22 @@ func (i *installer) InstallElemental(config elementalv1.Config) error { return nil } +func (i *installer) UpdateSystemAgentConfig(config elementalv1.Elemental) error { + agentConfPath, err := i.writeSystemAgentConfig(config) + if err != nil { + return fmt.Errorf("failed to write system agent configuration: %w", err) + } + config.Install.ConfigURLs = []string{agentConfPath} + installDataMap, err := structToMap(config.Install) + if err != nil { + return fmt.Errorf("failed to decode elemental-cli install data: %w", err) + } + if err := elementalcli.Run(installDataMap); err != nil { + return fmt.Errorf("failed to install elemental: %w", err) + } + return nil +} + func structToMap(str interface{}) (map[string]interface{}, error) { var mapStruct map[string]interface{} diff --git a/pkg/install/mocks/installer.go b/pkg/install/mocks/installer.go index 33a6560ce..c8de7cd3a 100644 --- a/pkg/install/mocks/installer.go +++ b/pkg/install/mocks/installer.go @@ -61,3 +61,17 @@ func (mr *MockInstallerMockRecorder) IsSystemInstalled() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSystemInstalled", reflect.TypeOf((*MockInstaller)(nil).IsSystemInstalled)) } + +// UpdateSystemAgentConfig mocks base method. +func (m *MockInstaller) UpdateSystemAgentConfig(arg0 v1beta1.Elemental) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateSystemAgentConfig", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateSystemAgentConfig indicates an expected call of UpdateSystemAgentConfig. +func (mr *MockInstallerMockRecorder) UpdateSystemAgentConfig(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSystemAgentConfig", reflect.TypeOf((*MockInstaller)(nil).UpdateSystemAgentConfig), arg0) +} From ee8291497548aa4ca689166b63711598d2f507f3 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 24 Jul 2023 14:43:24 +0200 Subject: [PATCH 02/50] Create reset secret on machine inventory deletion --- api/v1beta1/machineinventory_types.go | 4 + controllers/machineinventory_controller.go | 158 ++++++++++++++++++++- controllers/machineselector_controller.go | 1 + 3 files changed, 160 insertions(+), 3 deletions(-) diff --git a/api/v1beta1/machineinventory_types.go b/api/v1beta1/machineinventory_types.go index 9ffa67a35..9fd557106 100644 --- a/api/v1beta1/machineinventory_types.go +++ b/api/v1beta1/machineinventory_types.go @@ -24,6 +24,10 @@ import ( var ( MachineInventoryFinalizer = "machineinventory.elemental.cattle.io" PlanSecretType corev1.SecretType = "elemental.cattle.io/plan" + PlanTypeAnnotation = "elemental.cattle.io/plan.type" + PlanTypeEmpty = "empty" + PlanTypeBootstrap = "bootstrap" + PlanTypeReset = "reset" ) type MachineInventorySpec struct { diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 9d16053bd..3eebea4eb 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -17,11 +17,15 @@ limitations under the License. package controllers import ( + "bytes" "context" + "encoding/base64" + "encoding/json" "fmt" "time" "github.com/google/go-cmp/cmp" + "github.com/mudler/yip/pkg/schema" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -35,10 +39,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/yaml" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" "github.com/rancher/elemental-operator/pkg/log" "github.com/rancher/elemental-operator/pkg/util" + "github.com/rancher/system-agent/pkg/applyinator" ) // Timeout to validate machine inventory adoption @@ -112,7 +118,15 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * logger.Info("Reconciling machineinventory object") if mInventory.GetDeletionTimestamp() != nil { - controllerutil.RemoveFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) + if err := r.reconcileResetPlanSecret(ctx, mInventory); err != nil { + meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ + Type: elementalv1.ReadyCondition, + Reason: elementalv1.PlanFailureReason, + Status: metav1.ConditionFalse, + Message: err.Error(), + }) + return ctrl.Result{}, fmt.Errorf("failed to reconcile reset plan secret: %w", err) + } return ctrl.Result{}, nil } @@ -151,6 +165,143 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * return ctrl.Result{}, nil } +func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Context, mInventory *elementalv1.MachineInventory) error { + logger := ctrl.LoggerFrom(ctx) + + logger.Info("Reconciling Reset plan") + + secretName := mInventory.Name + secretNamespace := mInventory.Namespace + + foundSecret := &corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: secretNamespace}, foundSecret) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("looking up secret '%s' in namespace '%s': %w", secretName, secretNamespace, err) + } + + if apierrors.IsNotFound(err) { + logger.V(log.DebugDepth).Info("No existing secret found. Creating new reset plan secret.") + return r.createResetPlanSecret(ctx, mInventory) + } + + if err == nil { + if !util.IsObjectOwned(&foundSecret.ObjectMeta, mInventory.UID) { + return fmt.Errorf("secret already exists and was not created by this controller") + } + + planType, annotationFound := foundSecret.Annotations[elementalv1.PlanTypeAnnotation] + + if !annotationFound || planType != elementalv1.PlanTypeReset { + logger.V(log.DebugDepth).Info("Non reset plan type found. Replacing it with new reset plan secret.") + if err := r.Delete(ctx, foundSecret); err != nil { + return fmt.Errorf("deleting existing secret: %w", err) + } + return r.createResetPlanSecret(ctx, mInventory) + } + + logger.V(log.DebugDepth).Info("Reset plan type found. Updating status to determine whether it was successfully applied.") + if err := r.updateInventoryWithPlanStatus(ctx, mInventory); err != nil { + return fmt.Errorf("updating inventory with plan status: %w", err) + } + if mInventory.Status.Plan.State == elementalv1.PlanApplied { + logger.V(log.DebugDepth).Info("Reset plan was successfully applied.") + controllerutil.RemoveFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) + } + } + return nil +} + +func (r *MachineInventoryReconciler) createResetPlanSecret(ctx context.Context, mInventory *elementalv1.MachineInventory) error { + logger := ctrl.LoggerFrom(ctx) + + logger.Info("Creating new Reset plan secret") + + // This is the local cloud-config that the elemental-system-agent will run in recovery mode + resetCloudConfig := schema.YipConfig{ + Name: "Elemental Reset", + Stages: map[string][]schema.Stage{ + "network": { + schema.Stage{ + If: "'[ -f /run/cos/recovery_mode ]'", + Name: "Runs elemental reset", + Commands: []string{ + "elemental --debug reset --reset-persistent", + "rm -f /oem/reset-plan.yaml", + "reboot", + }, + }, + }, + }, + } + + resetCloudConfigBytes, err := yaml.Marshal(resetCloudConfig) + if err != nil { + return fmt.Errorf("marshalling local reset cloud-config to yaml: %w", err) + } + + // This is the remote plan that should trigger the reboot into recovery and reset + resetPlan := applyinator.Plan{ + Files: []applyinator.File{ + { + Content: base64.StdEncoding.EncodeToString(resetCloudConfigBytes), + Path: "/oem/reset-plan.yaml", + Permissions: "0600", + }, + }, + OneTimeInstructions: []applyinator.OneTimeInstruction{ + { + CommonInstruction: applyinator.CommonInstruction{ + Name: "configure next boot to recovery mode", + Command: "grub2-editenv", + Args: []string{ + "/oem/grubenv", + "set", + "next_entry=recovery", + }, + }, + }, + { + CommonInstruction: applyinator.CommonInstruction{ + Name: "reboot", + Command: "shutdown -r +1", // Postpone reboot, so the agent can mark the Plan as applied. + }, + }, + }, + } + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(resetPlan); err != nil { + return fmt.Errorf("failed to encode reset plan: %w", err) + } + + planSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeReset}, + Namespace: mInventory.Namespace, + Name: mInventory.Name, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: elementalv1.GroupVersion.String(), + Kind: "MachineInventory", + Name: mInventory.Name, + UID: mInventory.UID, + Controller: pointer.Bool(true), + }, + }, + Labels: map[string]string{ + elementalv1.ElementalManagedLabel: "true", + }, + }, + Type: elementalv1.PlanSecretType, + Data: map[string][]byte{"plan": buf.Bytes()}, + } + + if err := r.Create(ctx, planSecret); err != nil { + return fmt.Errorf("failed to create secret: %w", err) + } + return nil +} + func (r *MachineInventoryReconciler) createPlanSecret(ctx context.Context, mInventory *elementalv1.MachineInventory) error { logger := ctrl.LoggerFrom(ctx) @@ -162,8 +313,9 @@ func (r *MachineInventoryReconciler) createPlanSecret(ctx context.Context, mInve planSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Namespace: mInventory.Namespace, - Name: mInventory.Name, + Annotations: map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeEmpty}, + Namespace: mInventory.Namespace, + Name: mInventory.Name, OwnerReferences: []metav1.OwnerReference{ { APIVersion: elementalv1.GroupVersion.String(), diff --git a/controllers/machineselector_controller.go b/controllers/machineselector_controller.go index ee0c241f2..df189681d 100644 --- a/controllers/machineselector_controller.go +++ b/controllers/machineselector_controller.go @@ -360,6 +360,7 @@ func (r *MachineInventorySelectorReconciler) updatePlanSecretWithBootstrap(ctx c patchBase := client.MergeFrom(planSecret.DeepCopy()) planSecret.Data["plan"] = plan + planSecret.Annotations = map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeBootstrap} if err := r.Patch(ctx, planSecret, patchBase); err != nil { return fmt.Errorf("failed to patch plan secret: %w", err) From f23c9a2102f9001544fbf62f2d4bd4fa485786ec Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 26 Jul 2023 10:18:30 +0200 Subject: [PATCH 03/50] Patch plan secret with reset plan instead of recreating it --- controllers/machineinventory_controller.go | 136 +++++++++++---------- 1 file changed, 74 insertions(+), 62 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 3eebea4eb..4303c2fc8 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -125,7 +125,7 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * Status: metav1.ConditionFalse, Message: err.Error(), }) - return ctrl.Result{}, fmt.Errorf("failed to reconcile reset plan secret: %w", err) + return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) } return ctrl.Result{}, nil } @@ -170,53 +170,85 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex logger.Info("Reconciling Reset plan") - secretName := mInventory.Name - secretNamespace := mInventory.Namespace + if mInventory.Status.Plan == nil || mInventory.Status.Plan.PlanSecretRef == nil { + logger.V(log.DebugDepth).Info("Machine inventory plan reference not set yet. Creating new empty plan.") + return r.createPlanSecret(ctx, mInventory) // Recover from this unexpected state by creating a new empty plan secret + } - foundSecret := &corev1.Secret{} - err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: secretNamespace}, foundSecret) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("looking up secret '%s' in namespace '%s': %w", secretName, secretNamespace, err) + planSecret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{ + Namespace: mInventory.Status.Plan.PlanSecretRef.Namespace, + Name: mInventory.Status.Plan.PlanSecretRef.Name, + }, planSecret); err != nil { + return fmt.Errorf("getting plan secret: %w", err) } - if apierrors.IsNotFound(err) { - logger.V(log.DebugDepth).Info("No existing secret found. Creating new reset plan secret.") - return r.createResetPlanSecret(ctx, mInventory) + if !util.IsObjectOwned(&planSecret.ObjectMeta, mInventory.UID) { + return fmt.Errorf("secret already exists and was not created by this controller") } - if err == nil { - if !util.IsObjectOwned(&foundSecret.ObjectMeta, mInventory.UID) { - return fmt.Errorf("secret already exists and was not created by this controller") - } + planType, annotationFound := planSecret.Annotations[elementalv1.PlanTypeAnnotation] - planType, annotationFound := foundSecret.Annotations[elementalv1.PlanTypeAnnotation] + if !annotationFound || planType != elementalv1.PlanTypeReset { + logger.V(log.DebugDepth).Info("Non reset plan type found. Updating it with new reset plan.") + return r.updatePlanSecretWithReset(ctx, mInventory) + } - if !annotationFound || planType != elementalv1.PlanTypeReset { - logger.V(log.DebugDepth).Info("Non reset plan type found. Replacing it with new reset plan secret.") - if err := r.Delete(ctx, foundSecret); err != nil { - return fmt.Errorf("deleting existing secret: %w", err) - } - return r.createResetPlanSecret(ctx, mInventory) - } + logger.V(log.DebugDepth).Info("Reset plan type found. Updating status to determine whether it was successfully applied.") + if err := r.updateInventoryWithPlanStatus(ctx, mInventory); err != nil { + return fmt.Errorf("updating inventory with plan status: %w", err) + } + if mInventory.Status.Plan.State == elementalv1.PlanApplied { + logger.V(log.DebugDepth).Info("Reset plan was successfully applied.") + controllerutil.RemoveFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) + } - logger.V(log.DebugDepth).Info("Reset plan type found. Updating status to determine whether it was successfully applied.") - if err := r.updateInventoryWithPlanStatus(ctx, mInventory); err != nil { - return fmt.Errorf("updating inventory with plan status: %w", err) - } - if mInventory.Status.Plan.State == elementalv1.PlanApplied { - logger.V(log.DebugDepth).Info("Reset plan was successfully applied.") - controllerutil.RemoveFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) - } + return nil +} + +func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Context, mInventory *elementalv1.MachineInventory) error { + logger := ctrl.LoggerFrom(ctx) + + logger.Info("Updating Secret with Reset plan") + + planSecret := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{ + Namespace: mInventory.Status.Plan.PlanSecretRef.Namespace, + Name: mInventory.Status.Plan.PlanSecretRef.Name, + }, planSecret); err != nil { + return fmt.Errorf("getting plan secret: %w", err) + } + + resetPlan, err := r.newResetPlan(ctx) + if err != nil { + return fmt.Errorf("getting new reset plan: %w", err) } + + patchBase := client.MergeFrom(planSecret.DeepCopy()) + + planSecret.Data["plan"] = resetPlan + planSecret.Annotations = map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeReset} + + if err := r.Patch(ctx, planSecret, patchBase); err != nil { + return fmt.Errorf("patching plan secret: %w", err) + } + + meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ + Type: elementalv1.ReadyCondition, + Reason: elementalv1.WaitingForPlanReason, + Status: metav1.ConditionFalse, + Message: "waiting for reset plan to be applied", + }) + return nil } -func (r *MachineInventoryReconciler) createResetPlanSecret(ctx context.Context, mInventory *elementalv1.MachineInventory) error { +func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, error) { logger := ctrl.LoggerFrom(ctx) logger.Info("Creating new Reset plan secret") - // This is the local cloud-config that the elemental-system-agent will run in recovery mode + // This is the local cloud-config that the elemental-system-agent will run while in recovery mode resetCloudConfig := schema.YipConfig{ Name: "Elemental Reset", Stages: map[string][]schema.Stage{ @@ -225,8 +257,11 @@ func (r *MachineInventoryReconciler) createResetPlanSecret(ctx context.Context, If: "'[ -f /run/cos/recovery_mode ]'", Name: "Runs elemental reset", Commands: []string{ - "elemental --debug reset --reset-persistent", - "rm -f /oem/reset-plan.yaml", + "cp /oem/registration/config.yaml /tmp/registration-config.yaml", + "elemental --debug reset --reset-persistent --reset-oem", + "mkdir -p /oem/registration", + "mv /tmp/registration-config.yaml /oem/registration/config.yaml", + "elemental-register --debug --reset", "reboot", }, }, @@ -236,7 +271,7 @@ func (r *MachineInventoryReconciler) createResetPlanSecret(ctx context.Context, resetCloudConfigBytes, err := yaml.Marshal(resetCloudConfig) if err != nil { - return fmt.Errorf("marshalling local reset cloud-config to yaml: %w", err) + return nil, fmt.Errorf("marshalling local reset cloud-config to yaml: %w", err) } // This is the remote plan that should trigger the reboot into recovery and reset @@ -263,7 +298,7 @@ func (r *MachineInventoryReconciler) createResetPlanSecret(ctx context.Context, { CommonInstruction: applyinator.CommonInstruction{ Name: "reboot", - Command: "shutdown -r +1", // Postpone reboot, so the agent can mark the Plan as applied. + Command: "shutdown -r +1", // Need to have time to confirm plan execution before rebooting }, }, }, @@ -271,35 +306,12 @@ func (r *MachineInventoryReconciler) createResetPlanSecret(ctx context.Context, var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(resetPlan); err != nil { - return fmt.Errorf("failed to encode reset plan: %w", err) + return nil, fmt.Errorf("failed to encode plan: %w", err) } - planSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeReset}, - Namespace: mInventory.Namespace, - Name: mInventory.Name, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: elementalv1.GroupVersion.String(), - Kind: "MachineInventory", - Name: mInventory.Name, - UID: mInventory.UID, - Controller: pointer.Bool(true), - }, - }, - Labels: map[string]string{ - elementalv1.ElementalManagedLabel: "true", - }, - }, - Type: elementalv1.PlanSecretType, - Data: map[string][]byte{"plan": buf.Bytes()}, - } + plan := buf.Bytes() - if err := r.Create(ctx, planSecret); err != nil { - return fmt.Errorf("failed to create secret: %w", err) - } - return nil + return plan, nil } func (r *MachineInventoryReconciler) createPlanSecret(ctx context.Context, mInventory *elementalv1.MachineInventory) error { From 0fa6e2bbf7da326d9335603d369ebe0e2695c5a9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 26 Jul 2023 14:06:15 +0200 Subject: [PATCH 04/50] Implement elemental-register --reset --- pkg/install/_testdata/agent-plan.txt | 18 +++ pkg/install/_testdata/cloud-config.txt | 4 + pkg/install/install.go | 143 +++++++++------------- pkg/install/install_test.go | 158 +++++++++++++++++++++++++ pkg/install/mocks/installer.go | 12 +- 5 files changed, 241 insertions(+), 94 deletions(-) create mode 100644 pkg/install/_testdata/agent-plan.txt create mode 100644 pkg/install/_testdata/cloud-config.txt create mode 100644 pkg/install/install_test.go diff --git a/pkg/install/_testdata/agent-plan.txt b/pkg/install/_testdata/agent-plan.txt new file mode 100644 index 000000000..93e595e5b --- /dev/null +++ b/pkg/install/_testdata/agent-plan.txt @@ -0,0 +1,18 @@ +name: Elemental System Agent Configuration +stages: + initramfs: + - files: + - path: /var/lib/elemental/agent/elemental_connection.json + permissions: 384 + owner: 0 + group: 0 + content: '{"kubeConfig":"apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: SnVzdCBmb3IgdGVzdGluZw==\n server: https://127.0.0.1.sslip.io/k8s/cluster/local\n name: cluster\ncontexts:\n- context:\n cluster: cluster\n user: user\n name: context\ncurrent-context: context\nkind: Config\npreferences: {}\nusers:\n- name: user\n user:\n token: a test token\n","namespace":"a test namespace","secretName":"a test secret"}' + encoding: "" + ownerstring: "" + - path: /etc/rancher/elemental/agent/config.yaml + permissions: 384 + owner: 0 + group: 0 + content: '{"workDirectory":"/var/lib/elemental/agent/work","localEnabled":true,"localPlanDirectory":"/var/lib/elemental/agent/plans","appliedPlanDirectory":"/var/lib/elemental/agent/applied","remoteEnabled":true,"connectionInfoFile":"/var/lib/elemental/agent/elemental_connection.json"}' + encoding: "" + ownerstring: "" diff --git a/pkg/install/_testdata/cloud-config.txt b/pkg/install/_testdata/cloud-config.txt new file mode 100644 index 000000000..fb49c03df --- /dev/null +++ b/pkg/install/_testdata/cloud-config.txt @@ -0,0 +1,4 @@ +#cloud-config +users: +- name: root + passwd: root diff --git a/pkg/install/install.go b/pkg/install/install.go index 6dfa35c77..ea584450f 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -19,6 +19,7 @@ package install import ( "encoding/json" "fmt" + "os" "path/filepath" "github.com/mudler/yip/pkg/schema" @@ -28,7 +29,7 @@ import ( "github.com/rancher/elemental-operator/pkg/util" agent "github.com/rancher/system-agent/pkg/config" "github.com/twpayne/go-vfs" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" @@ -38,14 +39,15 @@ const ( stateInstallFile = "/run/initramfs/cos-state/state.yaml" agentStateDir = "/var/lib/elemental/agent" agentConfDir = "/etc/rancher/elemental/agent" - afterInstallHook = "/oem/install-hook.yaml" registrationConf = "/run/cos/oem/registration/config.yaml" + oemDir = "/oem" + oemDirLive = "/run/cos/oem" ) type Installer interface { IsSystemInstalled() bool InstallElemental(config elementalv1.Config) error - UpdateSystemAgentConfig(config elementalv1.Elemental) error + UpdateCloudConfig(config elementalv1.Config) error } func NewInstaller(fs vfs.FS) Installer { @@ -68,31 +70,6 @@ func (i *installer) IsSystemInstalled() bool { } func (i *installer) InstallElemental(config elementalv1.Config) error { - cloudInitURLs := config.Elemental.Install.ConfigURLs - if cloudInitURLs == nil { - cloudInitURLs = []string{} - } - - agentConfPath, err := i.writeSystemAgentConfig(config.Elemental) - if err != nil { - return fmt.Errorf("failed to write system agent configuration: %w", err) - } - cloudInitURLs = append(cloudInitURLs, agentConfPath) - - if len(config.CloudConfig) > 0 { - cloudInitPath, err := i.writeCloudInit(config.CloudConfig) - if err != nil { - return fmt.Errorf("failed to write custom cloud-init file: %w", err) - } - cloudInitURLs = append(cloudInitURLs, cloudInitPath) - } - - config.Elemental.Install.ConfigURLs = cloudInitURLs - - if err := i.installRegistrationYAML(config.Elemental.Registration); err != nil { - return fmt.Errorf("failed to prepare after-install hook: %w", err) - } - installDataMap, err := structToMap(config.Elemental.Install) if err != nil { return fmt.Errorf("failed to decode elemental-cli install data: %w", err) @@ -106,19 +83,34 @@ func (i *installer) InstallElemental(config elementalv1.Config) error { return nil } -func (i *installer) UpdateSystemAgentConfig(config elementalv1.Elemental) error { - agentConfPath, err := i.writeSystemAgentConfig(config) - if err != nil { - return fmt.Errorf("failed to write system agent configuration: %w", err) +func (i *installer) UpdateCloudConfig(config elementalv1.Config) error { + var baseDir string + if i.IsSystemInstalled() { + baseDir = oemDir + } else { + baseDir = oemDirLive } - config.Install.ConfigURLs = []string{agentConfPath} - installDataMap, err := structToMap(config.Install) - if err != nil { - return fmt.Errorf("failed to decode elemental-cli install data: %w", err) + + registrationConfigDir := fmt.Sprintf("%s/registration", baseDir) + if _, err := i.fs.Stat(registrationConfigDir); os.IsNotExist(err) { + log.Debugf("Registration config dir '%s' does not exist. Creating now.", registrationConfigDir) + if err := vfs.MkdirAll(i.fs, registrationConfigDir, 0700); err != nil { + return fmt.Errorf("creating registration config directory: %w", err) + } } - if err := elementalcli.Run(installDataMap); err != nil { - return fmt.Errorf("failed to install elemental: %w", err) + + if err := i.writeRegistrationConfig(config.Elemental.Registration, registrationConfigDir); err != nil { + return fmt.Errorf("writing registration config: %w", err) } + + if err := i.writeSystemAgentConfigPlan(config.Elemental, baseDir); err != nil { + return fmt.Errorf("writing system agent config plan: %w", err) + } + + if err := i.writeCloudInitConfig(config.CloudConfig, baseDir); err != nil { + return fmt.Errorf("writing cloud config: %w", err) + } + return nil } @@ -135,64 +127,38 @@ func structToMap(str interface{}) (map[string]interface{}, error) { return nil, err } -func (i *installer) installRegistrationYAML(reg elementalv1.Registration) error { +func (i *installer) writeRegistrationConfig(reg elementalv1.Registration, configDir string) error { registrationInBytes, err := yaml.Marshal(elementalv1.Config{ Elemental: elementalv1.Elemental{ Registration: reg, }, }) if err != nil { - return err + return fmt.Errorf("marshalling elemental configuration: %w", err) } - f, err := i.fs.Create(afterInstallHook) - if err != nil { - return err + configPath := fmt.Sprintf("%s/config.yaml", configDir) + if err := i.fs.WriteFile(configPath, registrationInBytes, os.FileMode(0600)); err != nil { + return fmt.Errorf("writing file '%s': %w", configPath, err) } - defer f.Close() - - err = yaml.NewEncoder(f).Encode(schema.YipConfig{ - Name: "Include registration config into installed system", - Stages: map[string][]schema.Stage{ - "after-install": { - schema.Stage{ - Directories: []schema.Directory{ - { - Path: filepath.Dir(registrationConf), - Permissions: 0700, - }, - }, Files: []schema.File{ - { - Path: registrationConf, - Content: string(registrationInBytes), - Permissions: 0600, - }, - }, - }, - }, - }, - }) - - return err + return nil } -func (i *installer) writeCloudInit(cloudConfig map[string]runtime.RawExtension) (string, error) { - f, err := i.fs.Create("/tmp/elemental-cloud-init.yaml") +func (i *installer) writeCloudInitConfig(cloudConfig map[string]runtime.RawExtension, configDir string) error { + cloudConfigBytes, err := util.MarshalCloudConfig(cloudConfig) if err != nil { - return "", fmt.Errorf("creating temporary cloud init file: %w", err) + return fmt.Errorf("mashalling cloud config: %w", err) } - defer f.Close() - bytes, err := util.MarshalCloudConfig(cloudConfig) - if err != nil { - return "", fmt.Errorf("mashalling cloud config: %w", err) - } + log.Debugf("Decoded CloudConfig:\n%s\n", string(cloudConfigBytes)) - log.Debugf("Decoded CloudConfig:\n%s\n", string(bytes)) - _, err = f.Write(bytes) - return f.Name(), err + cloudConfigPath := fmt.Sprintf("%s/cloud-config.yaml", configDir) + if err := i.fs.WriteFile(cloudConfigPath, cloudConfigBytes, os.FileMode(0600)); err != nil { + return fmt.Errorf("writing file '%s': %w", cloudConfigPath, err) + } + return nil } -func (i *installer) writeSystemAgentConfig(config elementalv1.Elemental) (string, error) { +func (i *installer) writeSystemAgentConfigPlan(config elementalv1.Elemental, configDir string) error { kubeConfig := api.Config{ Kind: "Config", APIVersion: "v1", @@ -251,18 +217,19 @@ func (i *installer) writeSystemAgentConfig(config elementalv1.Elemental) (string }, }) - f, err := i.fs.Create("/tmp/elemental-system-agent.yaml") - if err != nil { - return "", fmt.Errorf("creating temporary elemental-system-agent file: %w", err) - } - defer f.Close() - - err = yaml.NewEncoder(f).Encode(schema.YipConfig{ + planBytes, err := yaml.Marshal(schema.YipConfig{ Name: "Elemental System Agent Configuration", Stages: map[string][]schema.Stage{ "initramfs": stages, }, }) + if err != nil { + return fmt.Errorf("marshalling elemental system agent config plan: %w", err) + } - return f.Name(), err + planPath := fmt.Sprintf("%s/elemental-agent-config-plan.yaml", configDir) + if err := i.fs.WriteFile(planPath, planBytes, os.FileMode(0600)); err != nil { + return fmt.Errorf("writing file '%s': %w", planPath, err) + } + return nil } diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go new file mode 100644 index 000000000..5a2ed14a4 --- /dev/null +++ b/pkg/install/install_test.go @@ -0,0 +1,158 @@ +/* +Copyright © 2022 - 2023 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package install + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" + "github.com/twpayne/go-vfs" + "github.com/twpayne/go-vfs/vfst" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestInstall(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Install Suite") +} + +var ( + configFixture = elementalv1.Config{ + Elemental: elementalv1.Elemental{ + Registration: elementalv1.Registration{ + URL: "https://127.0.0.1.sslip.io", + CACert: "Just for testing", + EmulateTPM: true, + EmulatedTPMSeed: -1, + NoSMBIOS: true, + Auth: "test", + }, + SystemAgent: elementalv1.SystemAgent{ + URL: "https://127.0.0.1.sslip.io/k8s/cluster/local", + Token: "a test token", + SecretName: "a test secret", + SecretNamespace: "a test namespace", + }, + Install: elementalv1.Install{ + Firmware: "a test firmware", + Device: "a test device", + NoFormat: true, + ConfigURLs: []string{"foo", "bar"}, + ISO: "a test iso", + SystemURI: "a system uri", + Debug: true, + TTY: "a test tty", + PowerOff: true, + Reboot: true, + EjectCD: true, + DisableBootEntry: true, + ConfigDir: "a test config dir", + }, + }, + CloudConfig: map[string]runtime.RawExtension{ + "users": { + Raw: []byte(`[{"name":"root","passwd":"root"}]`), + }, + }, + } +) + +var _ = Describe("is system installed", Label("installation"), func() { + var fs vfs.FS + var err error + var fsCleanup func() + var installer Installer + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + installer = NewInstaller(fs) + DeferCleanup(fsCleanup) + }) + When("system is already installed", func() { + BeforeEach(func() { + Expect(vfs.MkdirAll(fs, filepath.Dir(stateInstallFile), 0700)).ToNot(HaveOccurred()) + Expect(fs.WriteFile(stateInstallFile, []byte("{}\n"), os.ModePerm)).ToNot(HaveOccurred()) + }) + It("should return system is installed", func() { + Expect(installer.IsSystemInstalled()).To(BeTrue()) + }) + }) + When("system is not installed", func() { + It("should return system is not installed", func() { + Expect(installer.IsSystemInstalled()).To(BeFalse()) + }) + }) +}) + +var _ = Describe("update cloud config", Label("installation", "cloud-config"), func() { + var fs vfs.FS + var err error + var fsCleanup func() + var installer Installer + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + installer = NewInstaller(fs) + DeferCleanup(fsCleanup) + }) + When("system is already installed", func() { + BeforeEach(func() { + Expect(vfs.MkdirAll(fs, filepath.Dir(stateInstallFile), 0700)).ToNot(HaveOccurred()) + Expect(fs.WriteFile(stateInstallFile, []byte("{}\n"), os.ModePerm)).ToNot(HaveOccurred()) + }) + It("should write config on /oem", func() { + Expect(installer.UpdateCloudConfig(configFixture)).ToNot(HaveOccurred()) + checkConfigInDir(fs, "/oem") + }) + }) + When("system is not installed", func() { + It("should write config on /run/cos/oem", func() { + Expect(installer.UpdateCloudConfig(configFixture)).ToNot(HaveOccurred()) + checkConfigInDir(fs, "/run/cos/oem") + }) + }) +}) + +func checkConfigInDir(fs vfs.FS, dir string) { + config := elementalv1.Config{} + registrationConfigFile, err := fs.ReadFile(fmt.Sprintf("%s/registration/config.yaml", dir)) + Expect(err).ToNot(HaveOccurred()) + Expect(yaml.Unmarshal(registrationConfigFile, &config)).ToNot(HaveOccurred()) + Expect(config).To(Equal(elementalv1.Config{ + Elemental: elementalv1.Elemental{ + Registration: configFixture.Elemental.Registration, + }, + })) + + systemAgentPlanRaw, err := fs.ReadFile(fmt.Sprintf("%s/elemental-agent-config-plan.yaml", dir)) + Expect(err).ToNot(HaveOccurred()) + wantSystemAgentPlan, err := ioutil.ReadFile("_testdata/agent-plan.txt") + Expect(err).ToNot(HaveOccurred()) + Expect(string(systemAgentPlanRaw)).To(Equal(string(wantSystemAgentPlan))) + + cloudConfigRaw, err := fs.ReadFile(fmt.Sprintf("%s/cloud-config.yaml", dir)) + Expect(err).ToNot(HaveOccurred()) + wantCloudConfig, err := ioutil.ReadFile("_testdata/cloud-config.txt") + Expect(string(cloudConfigRaw)).To(Equal(string(wantCloudConfig))) +} diff --git a/pkg/install/mocks/installer.go b/pkg/install/mocks/installer.go index c8de7cd3a..9afc83366 100644 --- a/pkg/install/mocks/installer.go +++ b/pkg/install/mocks/installer.go @@ -62,16 +62,16 @@ func (mr *MockInstallerMockRecorder) IsSystemInstalled() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSystemInstalled", reflect.TypeOf((*MockInstaller)(nil).IsSystemInstalled)) } -// UpdateSystemAgentConfig mocks base method. -func (m *MockInstaller) UpdateSystemAgentConfig(arg0 v1beta1.Elemental) error { +// UpdateCloudConfig mocks base method. +func (m *MockInstaller) UpdateCloudConfig(arg0 v1beta1.Config) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateSystemAgentConfig", arg0) + ret := m.ctrl.Call(m, "UpdateCloudConfig", arg0) ret0, _ := ret[0].(error) return ret0 } -// UpdateSystemAgentConfig indicates an expected call of UpdateSystemAgentConfig. -func (mr *MockInstallerMockRecorder) UpdateSystemAgentConfig(arg0 interface{}) *gomock.Call { +// UpdateCloudConfig indicates an expected call of UpdateCloudConfig. +func (mr *MockInstallerMockRecorder) UpdateCloudConfig(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSystemAgentConfig", reflect.TypeOf((*MockInstaller)(nil).UpdateSystemAgentConfig), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCloudConfig", reflect.TypeOf((*MockInstaller)(nil).UpdateCloudConfig), arg0) } From 5436781cdc722fe9f6afc3e84ca721a6ab7f9d10 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 26 Jul 2023 14:20:04 +0200 Subject: [PATCH 05/50] Implement elemental-register --reset --- cmd/register/main.go | 14 ++++++++------ cmd/register/main_test.go | 27 ++++++++++++++++----------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index 73f4899ba..596526e94 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -44,6 +44,7 @@ const ( var ( cfg elementalv1.Config debug bool + reset bool configPath string statePath string ) @@ -107,17 +108,17 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err := yaml.Unmarshal(data, &cfg); err != nil { return fmt.Errorf("parsing returned configuration: %w", err) } + // If --reset called explicity or this is a first installation, + // we need to update the cloud-config + if reset || !installer.IsSystemInstalled() { + log.Info("Resetting cloud config information") + installer.UpdateCloudConfig(cfg) + } // Install if !installer.IsSystemInstalled() { log.Info("Installing Elemental") return installer.InstallElemental(cfg) } - // If the System is already installed, we should update the elemental-system-agent config. - // In case of reset if we just registered to a new MachineInventory. - log.Debug("Updating Elemental System Agent") - if err := installer.UpdateSystemAgentConfig(cfg.Elemental); err != nil { - return fmt.Errorf("updating elemental-system-agent configuration: %w", err) - } return nil }, @@ -144,6 +145,7 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa cmd.Flags().StringVar(&statePath, "state-path", defaultStatePath, "The full path of the elemental-register config") cmd.PersistentFlags().BoolP("version", "v", false, "print version and exit") _ = viper.BindPFlag("version", cmd.PersistentFlags().Lookup("version")) + cmd.Flags().BoolVar(&debug, "reset", false, "Reset the cloud-config using the remote MachineRegistration") return cmd } diff --git a/cmd/register/main_test.go b/cmd/register/main_test.go index 35f522c1f..6aafb2b14 100644 --- a/cmd/register/main_test.go +++ b/cmd/register/main_test.go @@ -115,7 +115,6 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f client.EXPECT(). Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). Return(marshalToBytes(baseConfigFixture), nil) - installer.EXPECT().UpdateSystemAgentConfig(baseConfigFixture.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should overwrite the config values with passed arguments", func() { @@ -132,7 +131,6 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f client.EXPECT(). Register(wantConfig.Elemental.Registration, []byte(wantConfig.Elemental.Registration.CACert)). Return(marshalToBytes(wantConfig), nil) - installer.EXPECT().UpdateSystemAgentConfig(wantConfig.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should use config path argument", func() { @@ -142,7 +140,6 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f client.EXPECT(). Register(alternateConfigFixture.Elemental.Registration, []byte(alternateConfigFixture.Elemental.Registration.CACert)). Return(marshalToBytes(alternateConfigFixture), nil) - installer.EXPECT().UpdateSystemAgentConfig(alternateConfigFixture.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should skip registration if lastUpdate is recent", func() { @@ -165,7 +162,6 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f client.EXPECT(). Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). Return(marshalToBytes(baseConfigFixture), nil) - installer.EXPECT().UpdateSystemAgentConfig(baseConfigFixture.Elemental).Return(nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should use state path argument", func() { @@ -179,6 +175,14 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) + It("should reset cloud config if reset argument", func() { + cmd.SetArgs([]string{"--reset"}) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(baseConfigFixture), nil) + installer.EXPECT().UpdateCloudConfig(baseConfigFixture).Return(nil) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) }) }) When("system is not installed", func() { @@ -189,7 +193,6 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f mockCtrl = gomock.NewController(GinkgoT()) installer = imocks.NewMockInstaller(mockCtrl) installer.EXPECT().IsSystemInstalled().AnyTimes().Return(false) - installer.EXPECT().UpdateSystemAgentConfig(gomock.Any()).Times(0) // Only expect update if the system is already installed client = rmocks.NewMockClient(mockCtrl) cmd = newCommand(fs, client, register.NewFileStateHandler(fs), installer) DeferCleanup(fsCleanup) @@ -200,10 +203,11 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f }) It("should trigger install on first registration", func() { cmd.SetArgs([]string{}) + installer.EXPECT().UpdateCloudConfig(alternateConfigFixture).Return(nil) installer.EXPECT().InstallElemental(alternateConfigFixture).Return(nil) - returnedConfig, err := yaml.Marshal(alternateConfigFixture) - Expect(err).ToNot(HaveOccurred()) - client.EXPECT().Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)).Return(returnedConfig, nil) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(alternateConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should always trigger install on registration update", func() { @@ -213,10 +217,11 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f LastUpdate: time.Now(), } marshalIntoFile(fs, registrationState, defaultStatePath) + installer.EXPECT().UpdateCloudConfig(alternateConfigFixture).Return(nil) installer.EXPECT().InstallElemental(alternateConfigFixture).Return(nil) - returnedConfig, err := yaml.Marshal(alternateConfigFixture) - Expect(err).ToNot(HaveOccurred()) - client.EXPECT().Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)).Return(returnedConfig, nil) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(alternateConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) }) From fbdeda0b24ce728080905e6a5b019cdbfbf2f041 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 27 Jul 2023 10:19:16 +0200 Subject: [PATCH 06/50] Implement --reset and --install elemental-register arguments --- api/v1beta1/types.go | 24 +++ api/v1beta1/zz_generated.deepcopy.go | 21 ++ cmd/register/main.go | 70 +++--- cmd/register/main_test.go | 282 ++++++++++++++----------- pkg/elementalcli/elementalcli.go | 126 +++++------ pkg/install/_testdata/agent-plan.txt | 18 -- pkg/install/_testdata/cloud-config.txt | 4 - pkg/install/install.go | 171 ++++++++------- pkg/install/install_test.go | 158 -------------- pkg/install/mocks/installer.go | 26 +-- pkg/register/mocks/state.go | 78 +++++++ scripts/generate_mocks.sh | 1 + 12 files changed, 484 insertions(+), 495 deletions(-) delete mode 100644 pkg/install/_testdata/agent-plan.txt delete mode 100644 pkg/install/_testdata/cloud-config.txt delete mode 100644 pkg/install/install_test.go create mode 100644 pkg/register/mocks/state.go diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 2aeb45d68..7db3acb6e 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -49,6 +49,28 @@ type Install struct { ConfigDir string `json:"config-dir,omitempty" yaml:"config-dir,omitempty"` } +type Reset struct { + // +optional + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + // +optional + // +kubebuilder:default:=true + ResetPersistent bool `json:"reset-persistent,omitempty" yaml:"reset-persistent,omitempty"` + // +optional + // +kubebuilder:default:=true + ResetOEM bool `json:"reset-oem,omitempty" yaml:"reset-oem,omitempty"` + // +optional + ConfigURLs []string `json:"config-urls,omitempty" yaml:"config-urls,omitempty"` + // +optional + SystemURI string `json:"system-uri,omitempty" yaml:"system-uri,omitempty"` + // +optional + Debug bool `json:"debug,omitempty" yaml:"debug,omitempty"` + // +optional + PowerOff bool `json:"poweroff,omitempty" yaml:"poweroff,omitempty"` + // +optional + // +kubebuilder:default:=true + Reboot bool `json:"reboot,omitempty" yaml:"reboot,omitempty"` +} + type Registration struct { // +optional URL string `json:"url,omitempty" yaml:"url,omitempty" mapstructure:"url"` @@ -80,6 +102,8 @@ type Elemental struct { // +optional Install Install `json:"install,omitempty" yaml:"install,omitempty"` // +optional + Reset Reset `json:"reset,omitempty" yaml:"reset,omitempty"` + // +optional Registration Registration `json:"registration,omitempty" yaml:"registration,omitempty"` // +optional SystemAgent SystemAgent `json:"system-agent,omitempty" yaml:"system-agent,omitempty"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 4e61fc351..69abcf736 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -99,6 +99,7 @@ func (in *ContainerImage) DeepCopy() *ContainerImage { func (in *Elemental) DeepCopyInto(out *Elemental) { *out = *in in.Install.DeepCopyInto(&out.Install) + in.Reset.DeepCopyInto(&out.Reset) out.Registration = in.Registration out.SystemAgent = in.SystemAgent } @@ -957,6 +958,26 @@ func (in *Registration) DeepCopy() *Registration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Reset) DeepCopyInto(out *Reset) { + *out = *in + if in.ConfigURLs != nil { + in, out := &in.ConfigURLs, &out.ConfigURLs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Reset. +func (in *Reset) DeepCopy() *Reset { + if in == nil { + return nil + } + out := new(Reset) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SeedImage) DeepCopyInto(out *SeedImage) { *out = *in diff --git a/cmd/register/main.go b/cmd/register/main.go index 596526e94..9a464bfd1 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -42,11 +42,12 @@ const ( ) var ( - cfg elementalv1.Config - debug bool - reset bool - configPath string - statePath string + cfg elementalv1.Config + debug bool + reset bool + installation bool + configPath string + statePath string ) var ( @@ -84,21 +85,23 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa return fmt.Errorf("initializing configuration: %w", err) } // Determine if registration should execute or skip a cycle - if err := stateHandler.Init(statePath); err != nil { - return fmt.Errorf("initializing state handler on path '%s': %w", statePath, err) - } - if skip, err := shouldSkipRegistration(stateHandler, installer); err != nil { - return fmt.Errorf("determining if registration should run: %w", err) - } else if skip { - log.Info("Nothing to do") - return nil + if !installation && !reset { + if err := stateHandler.Init(statePath); err != nil { + return fmt.Errorf("initializing state handler on path '%s': %w", statePath, err) + } + if skip, err := shouldSkipRegistration(stateHandler); err != nil { + return fmt.Errorf("determining if registration should run: %w", err) + } else if skip { + log.Info("Nothing to do") + return nil + } } // Validate CA caCert, err := getRegistrationCA(fs, cfg) if err != nil { return fmt.Errorf("validating CA: %w", err) } - // Register + // Register (and fetch the remote MachineRegistration) data, err := client.Register(cfg.Elemental.Registration, caCert) if err != nil { return fmt.Errorf("registering machine: %w", err) @@ -108,17 +111,16 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err := yaml.Unmarshal(data, &cfg); err != nil { return fmt.Errorf("parsing returned configuration: %w", err) } - // If --reset called explicity or this is a first installation, - // we need to update the cloud-config - if reset || !installer.IsSystemInstalled() { - log.Info("Resetting cloud config information") - installer.UpdateCloudConfig(cfg) - } // Install - if !installer.IsSystemInstalled() { - log.Info("Installing Elemental") + if installation { + log.Info("Installing elemental") return installer.InstallElemental(cfg) } + // Reset + if reset { + log.Info("Resetting Elemental") + return installer.ResetElemental(cfg) + } return nil }, @@ -137,24 +139,27 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa cmd.Flags().StringVar(&cfg.Elemental.Registration.Auth, "auth", "tpm", "Registration authentication method") _ = viper.BindPFlag("elemental.registration.auth", cmd.Flags().Lookup("auth")) cmd.Flags().BoolVarP(&debug, "debug", "d", false, "Enable debug logging") - if installer.IsSystemInstalled() { - cmd.Flags().StringVar(&configPath, "config-path", defaultConfigPath, "The full path of the elemental-register config") - } else { - cmd.Flags().StringVar(&configPath, "config-path", defaultLiveConfigPath, "The full path of the elemental-register config") - } + cmd.Flags().StringVar(&configPath, "config-path", defaultConfigPath, "The full path of the elemental-register config") cmd.Flags().StringVar(&statePath, "state-path", defaultStatePath, "The full path of the elemental-register config") cmd.PersistentFlags().BoolP("version", "v", false, "print version and exit") _ = viper.BindPFlag("version", cmd.PersistentFlags().Lookup("version")) - cmd.Flags().BoolVar(&debug, "reset", false, "Reset the cloud-config using the remote MachineRegistration") + cmd.Flags().BoolVar(&reset, "reset", false, "Reset the machine to its original post-installation state") + cmd.Flags().BoolVar(&installation, "install", false, "Install a new machine") return cmd } func initConfig(fs vfs.FS) error { + log.Infof("Register version %s, commit %s, commit date %s", version.Version, version.Commit, version.CommitDate) + if installation && reset { + return errors.New("--install and --reset flags are mutually exclusive") + } if debug { log.EnableDebugLogging() } - log.Infof("Register version %s, commit %s, commit date %s", version.Version, version.Commit, version.CommitDate) - + // If we are installing from a live environment, the default config path must be updated + if installation && (configPath == defaultConfigPath) { + configPath = defaultLiveConfigPath + } // Use go-vfs afero compatibility layer (required by Viper) afs := vfsafero.NewAferoFS(fs) viper.SetFs(afs) @@ -171,10 +176,7 @@ func initConfig(fs vfs.FS) error { return nil } -func shouldSkipRegistration(stateHandler register.StateHandler, installer install.Installer) (bool, error) { - if !installer.IsSystemInstalled() { - return false, nil - } +func shouldSkipRegistration(stateHandler register.StateHandler) (bool, error) { state, err := stateHandler.Load() if err != nil { return false, fmt.Errorf("loading registration state") diff --git a/cmd/register/main_test.go b/cmd/register/main_test.go index 6aafb2b14..872377964 100644 --- a/cmd/register/main_test.go +++ b/cmd/register/main_test.go @@ -79,11 +79,20 @@ var ( DisableBootEntry: true, ConfigDir: "a test config dir", }, + Reset: elementalv1.Reset{ + Enabled: true, + ResetPersistent: false, + ResetOEM: false, + ConfigURLs: []string{"foo", "bar"}, + SystemURI: "a system uri", + PowerOff: true, + Reboot: true, + }, }, } ) -var _ = Describe("elemental-register arguments", Label("registration", "cli"), func() { +var _ = Describe("elemental-register", Label("registration", "cli"), func() { var fs vfs.FS var err error var fsCleanup func() @@ -91,139 +100,162 @@ var _ = Describe("elemental-register arguments", Label("registration", "cli"), f var mockCtrl *gomock.Controller var client *rmocks.MockClient var installer *imocks.MockInstaller - When("system is already installed", func() { + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + mockCtrl = gomock.NewController(GinkgoT()) + client = rmocks.NewMockClient(mockCtrl) + installer = imocks.NewMockInstaller(mockCtrl) + cmd = newCommand(fs, client, register.NewFileStateHandler(fs), installer) + DeferCleanup(fsCleanup) + }) + It("should return no error when printing version", func() { + cmd.SetArgs([]string{"--version"}) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) + When("using existing default config", func() { BeforeEach(func() { - fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) - Expect(err).ToNot(HaveOccurred()) - mockCtrl = gomock.NewController(GinkgoT()) - client = rmocks.NewMockClient(mockCtrl) - installer = imocks.NewMockInstaller(mockCtrl) - installer.EXPECT().IsSystemInstalled().AnyTimes().Return(true) - cmd = newCommand(fs, client, register.NewFileStateHandler(fs), installer) - DeferCleanup(fsCleanup) + marshalIntoFile(fs, baseConfigFixture, defaultConfigPath) }) - It("should return no error when printing version", func() { - cmd.SetArgs([]string{"--version"}) + It("should use the config if no arguments passed", func() { + cmd.SetArgs([]string{}) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(baseConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) - When("using existing default config", func() { - BeforeEach(func() { - marshalIntoFile(fs, baseConfigFixture, defaultConfigPath) - }) - It("should use the config if no arguments passed", func() { - cmd.SetArgs([]string{}) - client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). - Return(marshalToBytes(baseConfigFixture), nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) - It("should overwrite the config values with passed arguments", func() { - cmd.SetArgs([]string{ - "--registration-url", alternateConfigFixture.Elemental.Registration.URL, - "--registration-ca-cert", alternateConfigFixture.Elemental.Registration.CACert, - "--emulate-tpm", - "--emulated-tpm-seed", fmt.Sprintf("%d", alternateConfigFixture.Elemental.Registration.EmulatedTPMSeed), - "--no-smbios=false", - "--auth", alternateConfigFixture.Elemental.Registration.Auth, - }) - wantConfig := alternateConfigFixture.DeepCopy() - wantConfig.Elemental.Registration.NoSMBIOS = false - client.EXPECT(). - Register(wantConfig.Elemental.Registration, []byte(wantConfig.Elemental.Registration.CACert)). - Return(marshalToBytes(wantConfig), nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) - It("should use config path argument", func() { - newPath := "/a/custom/config/path/custom-config.yaml" - cmd.SetArgs([]string{"--config-path", newPath}) - marshalIntoFile(fs, alternateConfigFixture, newPath) - client.EXPECT(). - Register(alternateConfigFixture.Elemental.Registration, []byte(alternateConfigFixture.Elemental.Registration.CACert)). - Return(marshalToBytes(alternateConfigFixture), nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) - It("should skip registration if lastUpdate is recent", func() { - cmd.SetArgs([]string{}) - registrationState := register.State{ - InitialRegistration: time.Now(), - LastUpdate: time.Now(), - } - marshalIntoFile(fs, registrationState, defaultStatePath) - client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) - It("should not skip registration if lastUpdate is stale", func() { - cmd.SetArgs([]string{}) - registrationState := register.State{ - InitialRegistration: time.Now(), - LastUpdate: time.Now().Add(-25 * time.Hour), - } - marshalIntoFile(fs, registrationState, defaultStatePath) - client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). - Return(marshalToBytes(baseConfigFixture), nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) - It("should use state path argument", func() { - newPath := "/a/custom/state/path/custom-state.yaml" - cmd.SetArgs([]string{"--state-path", newPath}) - registrationState := register.State{ - InitialRegistration: time.Now(), - LastUpdate: time.Now(), - } - marshalIntoFile(fs, registrationState, newPath) - client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) - It("should reset cloud config if reset argument", func() { - cmd.SetArgs([]string{"--reset"}) - client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). - Return(marshalToBytes(baseConfigFixture), nil) - installer.EXPECT().UpdateCloudConfig(baseConfigFixture).Return(nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) + It("should overwrite the config values with passed arguments", func() { + cmd.SetArgs([]string{ + "--registration-url", alternateConfigFixture.Elemental.Registration.URL, + "--registration-ca-cert", alternateConfigFixture.Elemental.Registration.CACert, + "--emulate-tpm", + "--emulated-tpm-seed", fmt.Sprintf("%d", alternateConfigFixture.Elemental.Registration.EmulatedTPMSeed), + "--no-smbios=false", + "--auth", alternateConfigFixture.Elemental.Registration.Auth, }) + wantConfig := alternateConfigFixture.DeepCopy() + wantConfig.Elemental.Registration.NoSMBIOS = false + client.EXPECT(). + Register(wantConfig.Elemental.Registration, []byte(wantConfig.Elemental.Registration.CACert)). + Return(marshalToBytes(wantConfig), nil) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) + It("should use config path argument", func() { + newPath := "/a/custom/config/path/custom-config.yaml" + cmd.SetArgs([]string{"--config-path", newPath}) + marshalIntoFile(fs, alternateConfigFixture, newPath) + client.EXPECT(). + Register(alternateConfigFixture.Elemental.Registration, []byte(alternateConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(alternateConfigFixture), nil) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) + It("should skip registration if lastUpdate is recent", func() { + cmd.SetArgs([]string{}) + registrationState := register.State{ + InitialRegistration: time.Now(), + LastUpdate: time.Now(), + } + marshalIntoFile(fs, registrationState, defaultStatePath) + client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) + It("should not skip registration if lastUpdate is stale", func() { + cmd.SetArgs([]string{}) + registrationState := register.State{ + InitialRegistration: time.Now(), + LastUpdate: time.Now().Add(-25 * time.Hour), + } + marshalIntoFile(fs, registrationState, defaultStatePath) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(baseConfigFixture), nil) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) + It("should use state path argument", func() { + newPath := "/a/custom/state/path/custom-state.yaml" + cmd.SetArgs([]string{"--state-path", newPath}) + registrationState := register.State{ + InitialRegistration: time.Now(), + LastUpdate: time.Now(), + } + marshalIntoFile(fs, registrationState, newPath) + client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) + It("should reset cloud config if reset argument", func() { + cmd.SetArgs([]string{"--reset"}) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(baseConfigFixture), nil) + Expect(cmd.Execute()).ToNot(HaveOccurred()) + }) + }) +}) + +var _ = Describe("elemental-register --install", Label("registration", "cli", "install"), func() { + var fs vfs.FS + var err error + var fsCleanup func() + var cmd *cobra.Command + var mockCtrl *gomock.Controller + var client *rmocks.MockClient + var installer *imocks.MockInstaller + var stateHandler *rmocks.MockStateHandler + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + mockCtrl = gomock.NewController(GinkgoT()) + installer = imocks.NewMockInstaller(mockCtrl) + stateHandler = rmocks.NewMockStateHandler(mockCtrl) // Expect no calls + client = rmocks.NewMockClient(mockCtrl) + cmd = newCommand(fs, client, stateHandler, installer) + DeferCleanup(fsCleanup) + }) + When("using existing live config", func() { + BeforeEach(func() { + marshalIntoFile(fs, baseConfigFixture, defaultLiveConfigPath) + }) + It("should trigger install when --install argument", func() { + cmd.SetArgs([]string{"--install"}) + installer.EXPECT().InstallElemental(alternateConfigFixture).Return(nil) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(alternateConfigFixture), nil) + Expect(cmd.Execute()).ToNot(HaveOccurred()) }) }) - When("system is not installed", func() { - var installer *imocks.MockInstaller +}) + +var _ = Describe("elemental-register --reset", Label("registration", "cli", "reset"), func() { + var fs vfs.FS + var err error + var fsCleanup func() + var cmd *cobra.Command + var mockCtrl *gomock.Controller + var client *rmocks.MockClient + var installer *imocks.MockInstaller + var stateHandler *rmocks.MockStateHandler + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + mockCtrl = gomock.NewController(GinkgoT()) + installer = imocks.NewMockInstaller(mockCtrl) + stateHandler = rmocks.NewMockStateHandler(mockCtrl) // Expect no calls + client = rmocks.NewMockClient(mockCtrl) + cmd = newCommand(fs, client, stateHandler, installer) + DeferCleanup(fsCleanup) + }) + When("using existing default config", func() { BeforeEach(func() { - fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) - Expect(err).ToNot(HaveOccurred()) - mockCtrl = gomock.NewController(GinkgoT()) - installer = imocks.NewMockInstaller(mockCtrl) - installer.EXPECT().IsSystemInstalled().AnyTimes().Return(false) - client = rmocks.NewMockClient(mockCtrl) - cmd = newCommand(fs, client, register.NewFileStateHandler(fs), installer) - DeferCleanup(fsCleanup) + marshalIntoFile(fs, baseConfigFixture, defaultConfigPath) }) - When("using existing live config", func() { - BeforeEach(func() { - marshalIntoFile(fs, baseConfigFixture, defaultLiveConfigPath) - }) - It("should trigger install on first registration", func() { - cmd.SetArgs([]string{}) - installer.EXPECT().UpdateCloudConfig(alternateConfigFixture).Return(nil) - installer.EXPECT().InstallElemental(alternateConfigFixture).Return(nil) - client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). - Return(marshalToBytes(alternateConfigFixture), nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) - It("should always trigger install on registration update", func() { - cmd.SetArgs([]string{}) - registrationState := register.State{ - InitialRegistration: time.Now(), - LastUpdate: time.Now(), - } - marshalIntoFile(fs, registrationState, defaultStatePath) - installer.EXPECT().UpdateCloudConfig(alternateConfigFixture).Return(nil) - installer.EXPECT().InstallElemental(alternateConfigFixture).Return(nil) - client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). - Return(marshalToBytes(alternateConfigFixture), nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) + It("should trigger reset when --reset argument", func() { + cmd.SetArgs([]string{"--reset"}) + installer.EXPECT().ResetElemental(alternateConfigFixture).Return(nil) + client.EXPECT(). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Return(marshalToBytes(alternateConfigFixture), nil) + Expect(cmd.Execute()).ToNot(HaveOccurred()) }) }) }) diff --git a/pkg/elementalcli/elementalcli.go b/pkg/elementalcli/elementalcli.go index aeb1efc4e..556ca9067 100644 --- a/pkg/elementalcli/elementalcli.go +++ b/pkg/elementalcli/elementalcli.go @@ -20,36 +20,39 @@ import ( "fmt" "os" "os/exec" + "strconv" "strings" - "github.com/rancher/wrangler/pkg/data/convert" - - "github.com/rancher/elemental-operator/pkg/log" + elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" ) -const installMediaConfigDir = "/run/initramfs/live/elemental" +type Runner interface { + Install(elementalv1.Install) error + Reset(elementalv1.Reset) error +} + +func NewRunner() Runner { + return &runner{} +} + +var _ Runner = (*runner)(nil) -func Run(conf map[string]interface{}) error { - ev := mapToEnv("ELEMENTAL_INSTALL_", conf) +type runner struct{} +func (r *runner) Install(conf elementalv1.Install) error { installerOpts := []string{"elemental"} // There are no env var bindings in elemental-cli for elemental root options // so root flags should be passed within the command line - debug, ok := conf["debug"].(bool) - if ok && debug { + if conf.Debug { installerOpts = append(installerOpts, "--debug") } - configDir, ok := conf["config-dir"].(string) - if ok && configDir != "" { - installerOpts = append(installerOpts, "--config-dir", configDir) - } else { - log.Infof("Attempt to load elemental client config from default path: %s", installMediaConfigDir) - installerOpts = append(installerOpts, "--config-dir", installMediaConfigDir) + if conf.ConfigDir != "" { + installerOpts = append(installerOpts, "--config-dir", conf.ConfigDir) } installerOpts = append(installerOpts, "install") cmd := exec.Command("elemental") - cmd.Env = append(os.Environ(), ev...) + cmd.Env = append(os.Environ(), mapToInstallEnv(conf)...) cmd.Stdout = os.Stdout cmd.Args = installerOpts cmd.Stdin = os.Stdin @@ -57,54 +60,55 @@ func Run(conf map[string]interface{}) error { return cmd.Run() } -// it's a mapping of how config env option should be transliterated to the elemental CLI -var defaultOverrides = map[string]string{ - "ELEMENTAL_INSTALL_CONFIG_URLS": "ELEMENTAL_INSTALL_CLOUD_INIT", - "ELEMENTAL_INSTALL_POWEROFF": "ELEMENTAL_POWEROFF", - "ELEMENTAL_INSTALL_REBOOT": "ELEMENTAL_REBOOT", - "ELEMENTAL_INSTALL_EJECT_CD": "ELEMENTAL_EJECT_CD", - "ELEMENTAL_INSTALL_DEVICE": "ELEMENTAL_INSTALL_TARGET", - "ELEMENTAL_INSTALL_SYSTEM_URI": "ELEMENTAL_INSTALL_SYSTEM", - "ELEMENTAL_INSTALL_DEBUG": "ELEMENTAL_DEBUG", +func (r *runner) Reset(conf elementalv1.Reset) error { + installerOpts := []string{"elemental"} + // There are no env var bindings in elemental-cli for elemental root options + // so root flags should be passed within the command line + if conf.Debug { + installerOpts = append(installerOpts, "--debug") + } + installerOpts = append(installerOpts, "reset") + + cmd := exec.Command("elemental") + cmd.Env = append(os.Environ(), mapToResetEnv(conf)...) + cmd.Stdout = os.Stdout + cmd.Args = installerOpts + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + return cmd.Run() } -func envOverrides(keyName string) string { - for k, v := range defaultOverrides { - keyName = strings.ReplaceAll(keyName, k, v) - } - return keyName +func mapToInstallEnv(conf elementalv1.Install) []string { + var variables []string + // See GetInstallKeyEnvMap() in https://github.com/rancher/elemental-toolkit/blob/main/pkg/constants/constants.go + variables = append(variables, formatEV("ELEMENTAL_INSTALL_CLOUD_INIT", strings.Join(conf.ConfigURLs[:], ","))) + variables = append(variables, formatEV("ELEMENTAL_INSTALL_TARGET", conf.Device)) + variables = append(variables, formatEV("ELEMENTAL_INSTALL_SYSTEM", conf.SystemURI)) + variables = append(variables, formatEV("ELEMENTAL_INSTALL_FIRMWARE", conf.Firmware)) + variables = append(variables, formatEV("ELEMENTAL_INSTALL_ISO", conf.ISO)) + variables = append(variables, formatEV("ELEMENTAL_INSTALL_TTY", conf.TTY)) + variables = append(variables, formatEV("ELEMENTAL_INSTALL_DISABLE_BOOT_ENTRY", strconv.FormatBool(conf.DisableBootEntry))) + variables = append(variables, formatEV("ELEMENTAL_INSTALL_NO_FORMAT", strconv.FormatBool(conf.NoFormat))) + // See GetRunKeyEnvMap() in https://github.com/rancher/elemental-toolkit/blob/main/pkg/constants/constants.go + variables = append(variables, formatEV("ELEMENTAL_POWEROFF", strconv.FormatBool(conf.PowerOff))) + variables = append(variables, formatEV("ELEMENTAL_REBOOT", strconv.FormatBool(conf.Reboot))) + variables = append(variables, formatEV("ELEMENTAL_EJECT_CD", strconv.FormatBool(conf.EjectCD))) + return variables } -func mapToEnv(prefix string, data map[string]interface{}) []string { - var result []string - - log.Debugln("Computed environment variables:") - - for k, v := range data { - keyName := strings.ToUpper(prefix + convert.ToYAMLKey(k)) - keyName = strings.ReplaceAll(keyName, "-", "_") - // Apply overrides needed to convert between configs types - keyName = envOverrides(keyName) - - if data, ok := v.(map[string]interface{}); ok { - subResult := mapToEnv(keyName+"_", data) - result = append(result, subResult...) - } else if slice, ok := v.([]interface{}); ok { - // Convert slices into comma separated values, this is - // what viper/cobra support on elemental-cli side - ev := fmt.Sprintf("%s=", keyName) - for i, s := range slice { - if i < len(slice)-1 { - ev += fmt.Sprintf("%v,", s) - } else { - ev += fmt.Sprintf("%v", s) - } - } - result = append(result, ev) - } else { - result = append(result, fmt.Sprintf("%s=%v", keyName, v)) - log.Debugf("%s=%v\n", keyName, v) - } - } - return result +func mapToResetEnv(conf elementalv1.Reset) []string { + var variables []string + // See GetResetKeyEnvMap() in https://github.com/rancher/elemental-toolkit/blob/main/pkg/constants/constants.go + variables = append(variables, formatEV("ELEMENTAL_RESET_CLOUD_INIT", strings.Join(conf.ConfigURLs[:], ","))) + variables = append(variables, formatEV("ELEMENTAL_RESET_SYSTEM", conf.SystemURI)) + variables = append(variables, formatEV("ELEMENTAL_RESET_PERSISTENT", strconv.FormatBool(conf.ResetOEM))) + variables = append(variables, formatEV("ELEMENTAL_RESET_OEM", strconv.FormatBool(conf.ResetPersistent))) + // See GetRunKeyEnvMap() in https://github.com/rancher/elemental-toolkit/blob/main/pkg/constants/constants.go + variables = append(variables, formatEV("ELEMENTAL_POWEROFF", strconv.FormatBool(conf.PowerOff))) + variables = append(variables, formatEV("ELEMENTAL_REBOOT", strconv.FormatBool(conf.Reboot))) + return variables +} + +func formatEV(key string, value string) string { + return fmt.Sprintf("%s=%s", key, value) } diff --git a/pkg/install/_testdata/agent-plan.txt b/pkg/install/_testdata/agent-plan.txt deleted file mode 100644 index 93e595e5b..000000000 --- a/pkg/install/_testdata/agent-plan.txt +++ /dev/null @@ -1,18 +0,0 @@ -name: Elemental System Agent Configuration -stages: - initramfs: - - files: - - path: /var/lib/elemental/agent/elemental_connection.json - permissions: 384 - owner: 0 - group: 0 - content: '{"kubeConfig":"apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: SnVzdCBmb3IgdGVzdGluZw==\n server: https://127.0.0.1.sslip.io/k8s/cluster/local\n name: cluster\ncontexts:\n- context:\n cluster: cluster\n user: user\n name: context\ncurrent-context: context\nkind: Config\npreferences: {}\nusers:\n- name: user\n user:\n token: a test token\n","namespace":"a test namespace","secretName":"a test secret"}' - encoding: "" - ownerstring: "" - - path: /etc/rancher/elemental/agent/config.yaml - permissions: 384 - owner: 0 - group: 0 - content: '{"workDirectory":"/var/lib/elemental/agent/work","localEnabled":true,"localPlanDirectory":"/var/lib/elemental/agent/plans","appliedPlanDirectory":"/var/lib/elemental/agent/applied","remoteEnabled":true,"connectionInfoFile":"/var/lib/elemental/agent/elemental_connection.json"}' - encoding: "" - ownerstring: "" diff --git a/pkg/install/_testdata/cloud-config.txt b/pkg/install/_testdata/cloud-config.txt deleted file mode 100644 index fb49c03df..000000000 --- a/pkg/install/_testdata/cloud-config.txt +++ /dev/null @@ -1,4 +0,0 @@ -#cloud-config -users: -- name: root - passwd: root diff --git a/pkg/install/install.go b/pkg/install/install.go index ea584450f..eb0748de7 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -19,7 +19,6 @@ package install import ( "encoding/json" "fmt" - "os" "path/filepath" "github.com/mudler/yip/pkg/schema" @@ -29,7 +28,7 @@ import ( "github.com/rancher/elemental-operator/pkg/util" agent "github.com/rancher/system-agent/pkg/config" "github.com/twpayne/go-vfs" - "gopkg.in/yaml.v3" + "gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" @@ -39,126 +38,149 @@ const ( stateInstallFile = "/run/initramfs/cos-state/state.yaml" agentStateDir = "/var/lib/elemental/agent" agentConfDir = "/etc/rancher/elemental/agent" - registrationConf = "/run/cos/oem/registration/config.yaml" - oemDir = "/oem" - oemDirLive = "/run/cos/oem" + registrationConf = "/oem/registration/config.yaml" ) type Installer interface { - IsSystemInstalled() bool + ResetElemental(config elementalv1.Config) error InstallElemental(config elementalv1.Config) error - UpdateCloudConfig(config elementalv1.Config) error } func NewInstaller(fs vfs.FS) Installer { return &installer{ - fs: fs, + fs: fs, + runner: elementalcli.NewRunner(), } } var _ Installer = (*installer)(nil) type installer struct { - fs vfs.FS -} - -// IsSystemInstalled checks if the host is currently installed -// TODO: make the function dependent on tmp.Register returned data -func (i *installer) IsSystemInstalled() bool { - _, err := i.fs.Stat(stateInstallFile) - return err == nil + fs vfs.FS + runner elementalcli.Runner } func (i *installer) InstallElemental(config elementalv1.Config) error { - installDataMap, err := structToMap(config.Elemental.Install) + if config.Elemental.Install.ConfigURLs == nil { + config.Elemental.Install.ConfigURLs = []string{} + } + + additionalConfigs, err := i.getCloudInitConfigs(config) if err != nil { - return fmt.Errorf("failed to decode elemental-cli install data: %w", err) + return fmt.Errorf("generating additional cloud configs: %w", err) } + config.Elemental.Install.ConfigURLs = append(config.Elemental.Install.ConfigURLs, additionalConfigs...) - if err := elementalcli.Run(installDataMap); err != nil { + if err := i.runner.Install(config.Elemental.Install); err != nil { return fmt.Errorf("failed to install elemental: %w", err) } - log.Info("Elemental installation completed, please reboot") + log.Info("Elemental install completed, please reboot") return nil } -func (i *installer) UpdateCloudConfig(config elementalv1.Config) error { - var baseDir string - if i.IsSystemInstalled() { - baseDir = oemDir - } else { - baseDir = oemDirLive - } - - registrationConfigDir := fmt.Sprintf("%s/registration", baseDir) - if _, err := i.fs.Stat(registrationConfigDir); os.IsNotExist(err) { - log.Debugf("Registration config dir '%s' does not exist. Creating now.", registrationConfigDir) - if err := vfs.MkdirAll(i.fs, registrationConfigDir, 0700); err != nil { - return fmt.Errorf("creating registration config directory: %w", err) - } - } - - if err := i.writeRegistrationConfig(config.Elemental.Registration, registrationConfigDir); err != nil { - return fmt.Errorf("writing registration config: %w", err) +func (i *installer) ResetElemental(config elementalv1.Config) error { + if config.Elemental.Reset.ConfigURLs == nil { + config.Elemental.Reset.ConfigURLs = []string{} } - if err := i.writeSystemAgentConfigPlan(config.Elemental, baseDir); err != nil { - return fmt.Errorf("writing system agent config plan: %w", err) + additionalConfigs, err := i.getCloudInitConfigs(config) + if err != nil { + return fmt.Errorf("generating additional cloud configs: %w", err) } + config.Elemental.Reset.ConfigURLs = append(config.Elemental.Reset.ConfigURLs, additionalConfigs...) - if err := i.writeCloudInitConfig(config.CloudConfig, baseDir); err != nil { - return fmt.Errorf("writing cloud config: %w", err) + if err := i.runner.Reset(config.Elemental.Reset); err != nil { + return fmt.Errorf("failed to reset elemental: %w", err) } + log.Info("Elemental reset completed, please reboot") return nil } -func structToMap(str interface{}) (map[string]interface{}, error) { - var mapStruct map[string]interface{} +func (i *installer) getCloudInitConfigs(config elementalv1.Config) ([]string, error) { + configs := []string{} + agentConfPath, err := i.writeSystemAgentConfig(config.Elemental) + if err != nil { + return nil, fmt.Errorf("writing system agent configuration: %w", err) + } + configs = append(configs, agentConfPath) - data, err := json.Marshal(str) - if err == nil { - if err := json.Unmarshal(data, &mapStruct); err == nil { - return mapStruct, nil + if len(config.CloudConfig) > 0 { + cloudInitPath, err := i.writeCloudInit(config.CloudConfig) + if err != nil { + return nil, fmt.Errorf("writing custom cloud-init file: %w", err) } + configs = append(configs, cloudInitPath) } - return nil, err + registrationConfPath, err := i.writeRegistrationYAML(config.Elemental.Registration) + if err != nil { + return nil, fmt.Errorf("writing registration conf plan: %w", err) + } + configs = append(configs, registrationConfPath) + + return configs, nil } -func (i *installer) writeRegistrationConfig(reg elementalv1.Registration, configDir string) error { +func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, error) { + f, err := i.fs.Create("/tmp/elemental-registration-conf.yaml") + if err != nil { + return "", fmt.Errorf("creating temporary registration conf plan file: %w", err) + } + defer f.Close() registrationInBytes, err := yaml.Marshal(elementalv1.Config{ Elemental: elementalv1.Elemental{ Registration: reg, }, }) if err != nil { - return fmt.Errorf("marshalling elemental configuration: %w", err) - } - configPath := fmt.Sprintf("%s/config.yaml", configDir) - if err := i.fs.WriteFile(configPath, registrationInBytes, os.FileMode(0600)); err != nil { - return fmt.Errorf("writing file '%s': %w", configPath, err) + return "", err } - return nil + + err = yaml.NewEncoder(f).Encode(schema.YipConfig{ + Name: "Include registration config into installed system", + Stages: map[string][]schema.Stage{ + "initramfs": { + schema.Stage{ + Directories: []schema.Directory{ + { + Path: filepath.Dir(registrationConf), + Permissions: 0700, + }, + }, Files: []schema.File{ + { + Path: registrationConf, + Content: string(registrationInBytes), + Permissions: 0600, + }, + }, + }, + }, + }, + }) + + return f.Name(), err } -func (i *installer) writeCloudInitConfig(cloudConfig map[string]runtime.RawExtension, configDir string) error { - cloudConfigBytes, err := util.MarshalCloudConfig(cloudConfig) +func (i *installer) writeCloudInit(cloudConfig map[string]runtime.RawExtension) (string, error) { + f, err := i.fs.Create("/tmp/elemental-cloud-init.yaml") if err != nil { - return fmt.Errorf("mashalling cloud config: %w", err) + return "", fmt.Errorf("creating temporary cloud init file: %w", err) } + defer f.Close() - log.Debugf("Decoded CloudConfig:\n%s\n", string(cloudConfigBytes)) - - cloudConfigPath := fmt.Sprintf("%s/cloud-config.yaml", configDir) - if err := i.fs.WriteFile(cloudConfigPath, cloudConfigBytes, os.FileMode(0600)); err != nil { - return fmt.Errorf("writing file '%s': %w", cloudConfigPath, err) + bytes, err := util.MarshalCloudConfig(cloudConfig) + if err != nil { + return "", fmt.Errorf("mashalling cloud config: %w", err) } - return nil + + log.Debugf("Decoded CloudConfig:\n%s\n", string(bytes)) + _, err = f.Write(bytes) + return f.Name(), err } -func (i *installer) writeSystemAgentConfigPlan(config elementalv1.Elemental, configDir string) error { +func (i *installer) writeSystemAgentConfig(config elementalv1.Elemental) (string, error) { kubeConfig := api.Config{ Kind: "Config", APIVersion: "v1", @@ -217,19 +239,18 @@ func (i *installer) writeSystemAgentConfigPlan(config elementalv1.Elemental, con }, }) - planBytes, err := yaml.Marshal(schema.YipConfig{ + f, err := i.fs.Create("/tmp/elemental-system-agent.yaml") + if err != nil { + return "", fmt.Errorf("creating temporary elemental-system-agent file: %w", err) + } + defer f.Close() + + err = yaml.NewEncoder(f).Encode(schema.YipConfig{ Name: "Elemental System Agent Configuration", Stages: map[string][]schema.Stage{ "initramfs": stages, }, }) - if err != nil { - return fmt.Errorf("marshalling elemental system agent config plan: %w", err) - } - planPath := fmt.Sprintf("%s/elemental-agent-config-plan.yaml", configDir) - if err := i.fs.WriteFile(planPath, planBytes, os.FileMode(0600)); err != nil { - return fmt.Errorf("writing file '%s': %w", planPath, err) - } - return nil + return f.Name(), err } diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go deleted file mode 100644 index 5a2ed14a4..000000000 --- a/pkg/install/install_test.go +++ /dev/null @@ -1,158 +0,0 @@ -/* -Copyright © 2022 - 2023 SUSE LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package install - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" - "github.com/twpayne/go-vfs" - "github.com/twpayne/go-vfs/vfst" - "gopkg.in/yaml.v3" - "k8s.io/apimachinery/pkg/runtime" -) - -func TestInstall(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Install Suite") -} - -var ( - configFixture = elementalv1.Config{ - Elemental: elementalv1.Elemental{ - Registration: elementalv1.Registration{ - URL: "https://127.0.0.1.sslip.io", - CACert: "Just for testing", - EmulateTPM: true, - EmulatedTPMSeed: -1, - NoSMBIOS: true, - Auth: "test", - }, - SystemAgent: elementalv1.SystemAgent{ - URL: "https://127.0.0.1.sslip.io/k8s/cluster/local", - Token: "a test token", - SecretName: "a test secret", - SecretNamespace: "a test namespace", - }, - Install: elementalv1.Install{ - Firmware: "a test firmware", - Device: "a test device", - NoFormat: true, - ConfigURLs: []string{"foo", "bar"}, - ISO: "a test iso", - SystemURI: "a system uri", - Debug: true, - TTY: "a test tty", - PowerOff: true, - Reboot: true, - EjectCD: true, - DisableBootEntry: true, - ConfigDir: "a test config dir", - }, - }, - CloudConfig: map[string]runtime.RawExtension{ - "users": { - Raw: []byte(`[{"name":"root","passwd":"root"}]`), - }, - }, - } -) - -var _ = Describe("is system installed", Label("installation"), func() { - var fs vfs.FS - var err error - var fsCleanup func() - var installer Installer - BeforeEach(func() { - fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) - Expect(err).ToNot(HaveOccurred()) - installer = NewInstaller(fs) - DeferCleanup(fsCleanup) - }) - When("system is already installed", func() { - BeforeEach(func() { - Expect(vfs.MkdirAll(fs, filepath.Dir(stateInstallFile), 0700)).ToNot(HaveOccurred()) - Expect(fs.WriteFile(stateInstallFile, []byte("{}\n"), os.ModePerm)).ToNot(HaveOccurred()) - }) - It("should return system is installed", func() { - Expect(installer.IsSystemInstalled()).To(BeTrue()) - }) - }) - When("system is not installed", func() { - It("should return system is not installed", func() { - Expect(installer.IsSystemInstalled()).To(BeFalse()) - }) - }) -}) - -var _ = Describe("update cloud config", Label("installation", "cloud-config"), func() { - var fs vfs.FS - var err error - var fsCleanup func() - var installer Installer - BeforeEach(func() { - fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) - Expect(err).ToNot(HaveOccurred()) - installer = NewInstaller(fs) - DeferCleanup(fsCleanup) - }) - When("system is already installed", func() { - BeforeEach(func() { - Expect(vfs.MkdirAll(fs, filepath.Dir(stateInstallFile), 0700)).ToNot(HaveOccurred()) - Expect(fs.WriteFile(stateInstallFile, []byte("{}\n"), os.ModePerm)).ToNot(HaveOccurred()) - }) - It("should write config on /oem", func() { - Expect(installer.UpdateCloudConfig(configFixture)).ToNot(HaveOccurred()) - checkConfigInDir(fs, "/oem") - }) - }) - When("system is not installed", func() { - It("should write config on /run/cos/oem", func() { - Expect(installer.UpdateCloudConfig(configFixture)).ToNot(HaveOccurred()) - checkConfigInDir(fs, "/run/cos/oem") - }) - }) -}) - -func checkConfigInDir(fs vfs.FS, dir string) { - config := elementalv1.Config{} - registrationConfigFile, err := fs.ReadFile(fmt.Sprintf("%s/registration/config.yaml", dir)) - Expect(err).ToNot(HaveOccurred()) - Expect(yaml.Unmarshal(registrationConfigFile, &config)).ToNot(HaveOccurred()) - Expect(config).To(Equal(elementalv1.Config{ - Elemental: elementalv1.Elemental{ - Registration: configFixture.Elemental.Registration, - }, - })) - - systemAgentPlanRaw, err := fs.ReadFile(fmt.Sprintf("%s/elemental-agent-config-plan.yaml", dir)) - Expect(err).ToNot(HaveOccurred()) - wantSystemAgentPlan, err := ioutil.ReadFile("_testdata/agent-plan.txt") - Expect(err).ToNot(HaveOccurred()) - Expect(string(systemAgentPlanRaw)).To(Equal(string(wantSystemAgentPlan))) - - cloudConfigRaw, err := fs.ReadFile(fmt.Sprintf("%s/cloud-config.yaml", dir)) - Expect(err).ToNot(HaveOccurred()) - wantCloudConfig, err := ioutil.ReadFile("_testdata/cloud-config.txt") - Expect(string(cloudConfigRaw)).To(Equal(string(wantCloudConfig))) -} diff --git a/pkg/install/mocks/installer.go b/pkg/install/mocks/installer.go index 9afc83366..98bd6d113 100644 --- a/pkg/install/mocks/installer.go +++ b/pkg/install/mocks/installer.go @@ -48,30 +48,16 @@ func (mr *MockInstallerMockRecorder) InstallElemental(arg0 interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallElemental", reflect.TypeOf((*MockInstaller)(nil).InstallElemental), arg0) } -// IsSystemInstalled mocks base method. -func (m *MockInstaller) IsSystemInstalled() bool { +// ResetElemental mocks base method. +func (m *MockInstaller) ResetElemental(arg0 v1beta1.Config) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsSystemInstalled") - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsSystemInstalled indicates an expected call of IsSystemInstalled. -func (mr *MockInstallerMockRecorder) IsSystemInstalled() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSystemInstalled", reflect.TypeOf((*MockInstaller)(nil).IsSystemInstalled)) -} - -// UpdateCloudConfig mocks base method. -func (m *MockInstaller) UpdateCloudConfig(arg0 v1beta1.Config) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateCloudConfig", arg0) + ret := m.ctrl.Call(m, "ResetElemental", arg0) ret0, _ := ret[0].(error) return ret0 } -// UpdateCloudConfig indicates an expected call of UpdateCloudConfig. -func (mr *MockInstallerMockRecorder) UpdateCloudConfig(arg0 interface{}) *gomock.Call { +// ResetElemental indicates an expected call of ResetElemental. +func (mr *MockInstallerMockRecorder) ResetElemental(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateCloudConfig", reflect.TypeOf((*MockInstaller)(nil).UpdateCloudConfig), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetElemental", reflect.TypeOf((*MockInstaller)(nil).ResetElemental), arg0) } diff --git a/pkg/register/mocks/state.go b/pkg/register/mocks/state.go new file mode 100644 index 000000000..a16f2a8eb --- /dev/null +++ b/pkg/register/mocks/state.go @@ -0,0 +1,78 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/rancher/elemental-operator/pkg/register (interfaces: StateHandler) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + register "github.com/rancher/elemental-operator/pkg/register" + gomock "go.uber.org/mock/gomock" +) + +// MockStateHandler is a mock of StateHandler interface. +type MockStateHandler struct { + ctrl *gomock.Controller + recorder *MockStateHandlerMockRecorder +} + +// MockStateHandlerMockRecorder is the mock recorder for MockStateHandler. +type MockStateHandlerMockRecorder struct { + mock *MockStateHandler +} + +// NewMockStateHandler creates a new mock instance. +func NewMockStateHandler(ctrl *gomock.Controller) *MockStateHandler { + mock := &MockStateHandler{ctrl: ctrl} + mock.recorder = &MockStateHandlerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStateHandler) EXPECT() *MockStateHandlerMockRecorder { + return m.recorder +} + +// Init mocks base method. +func (m *MockStateHandler) Init(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockStateHandlerMockRecorder) Init(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockStateHandler)(nil).Init), arg0) +} + +// Load mocks base method. +func (m *MockStateHandler) Load() (register.State, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load") + ret0, _ := ret[0].(register.State) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Load indicates an expected call of Load. +func (mr *MockStateHandlerMockRecorder) Load() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockStateHandler)(nil).Load)) +} + +// Save mocks base method. +func (m *MockStateHandler) Save(arg0 register.State) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Save indicates an expected call of Save. +func (mr *MockStateHandlerMockRecorder) Save(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockStateHandler)(nil).Save), arg0) +} diff --git a/scripts/generate_mocks.sh b/scripts/generate_mocks.sh index 75a59adc3..4e7db6c50 100755 --- a/scripts/generate_mocks.sh +++ b/scripts/generate_mocks.sh @@ -6,4 +6,5 @@ go install go.uber.org/mock/mockgen@latest # See codecov.yml for more info mockgen -destination=pkg/register/mocks/client.go -package=mocks github.com/rancher/elemental-operator/pkg/register Client +mockgen -destination=pkg/register/mocks/state.go -package=mocks github.com/rancher/elemental-operator/pkg/register StateHandler mockgen -destination=pkg/install/mocks/installer.go -package=mocks github.com/rancher/elemental-operator/pkg/install Installer From 7e2604e85f7df2c122c55e07bb959ff6e4506cb5 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 27 Jul 2023 10:29:33 +0200 Subject: [PATCH 07/50] Add resettable annotation on MachineInventories --- api/v1beta1/machineinventory_types.go | 13 +++++++------ controllers/machineinventory_controller.go | 7 +++++++ pkg/server/server.go | 8 ++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/api/v1beta1/machineinventory_types.go b/api/v1beta1/machineinventory_types.go index 9fd557106..d8f4d077b 100644 --- a/api/v1beta1/machineinventory_types.go +++ b/api/v1beta1/machineinventory_types.go @@ -22,12 +22,13 @@ import ( ) var ( - MachineInventoryFinalizer = "machineinventory.elemental.cattle.io" - PlanSecretType corev1.SecretType = "elemental.cattle.io/plan" - PlanTypeAnnotation = "elemental.cattle.io/plan.type" - PlanTypeEmpty = "empty" - PlanTypeBootstrap = "bootstrap" - PlanTypeReset = "reset" + MachineInventoryFinalizer = "machineinventory.elemental.cattle.io" + PlanSecretType corev1.SecretType = "elemental.cattle.io/plan" + PlanTypeAnnotation = "elemental.cattle.io/plan.type" + PlanTypeEmpty = "empty" + PlanTypeBootstrap = "bootstrap" + PlanTypeReset = "reset" + MachineInventoryResettableAnnotation = "elemental.cattle.io/resettable" ) type MachineInventorySpec struct { diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 4303c2fc8..25f69aa18 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -170,6 +170,13 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex logger.Info("Reconciling Reset plan") + resettable, found := mInventory.Annotations[elementalv1.MachineInventoryResettableAnnotation] + if !found || resettable != "true" { + logger.V(log.DebugDepth).Info("Machine Inventory does not need reset. Removing finalizer.") + controllerutil.RemoveFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) + return nil + } + if mInventory.Status.Plan == nil || mInventory.Status.Plan.PlanSecretRef == nil { logger.V(log.DebugDepth).Info("Machine inventory plan reference not set yet. Creating new empty plan.") return r.createPlanSecret(ctx, mInventory) // Recover from this unexpected state by creating a new empty plan secret diff --git a/pkg/server/server.go b/pkg/server/server.go index 0cb326f4c..043cf60b1 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -161,6 +161,14 @@ func initInventory(inventory *elementalv1.MachineInventory, registration *elemen value, _ := replaceStringData(map[string]interface{}{}, v) inventory.Labels[k] = strings.TrimSuffix(strings.TrimPrefix(value, "-"), "-") } + + // Set resettable annotation on cascade from MachineRegistration spec + if registration.Spec.Config.Elemental.Reset.Enabled { + if inventory.Annotations == nil { + inventory.Annotations = map[string]string{} + } + inventory.Annotations[elementalv1.MachineInventoryResettableAnnotation] = "true" + } } func (i *InventoryServer) createMachineInventory(inventory *elementalv1.MachineInventory) (*elementalv1.MachineInventory, error) { From c0ba8fb86b64c1889525b2d4429f3c5e25d5d7a1 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 27 Jul 2023 11:10:40 +0200 Subject: [PATCH 08/50] Remove now redundant test --- cmd/register/main_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cmd/register/main_test.go b/cmd/register/main_test.go index 872377964..89c8e3e60 100644 --- a/cmd/register/main_test.go +++ b/cmd/register/main_test.go @@ -182,13 +182,6 @@ var _ = Describe("elemental-register", Label("registration", "cli"), func() { client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) - It("should reset cloud config if reset argument", func() { - cmd.SetArgs([]string{"--reset"}) - client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). - Return(marshalToBytes(baseConfigFixture), nil) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) }) }) From 1a183796f42716726b0dd82d4c21f1039f49da38 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 27 Jul 2023 11:15:47 +0200 Subject: [PATCH 09/50] Update reset plan --- controllers/machineinventory_controller.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 25f69aa18..908b1036e 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -264,12 +264,7 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, If: "'[ -f /run/cos/recovery_mode ]'", Name: "Runs elemental reset", Commands: []string{ - "cp /oem/registration/config.yaml /tmp/registration-config.yaml", - "elemental --debug reset --reset-persistent --reset-oem", - "mkdir -p /oem/registration", - "mv /tmp/registration-config.yaml /oem/registration/config.yaml", "elemental-register --debug --reset", - "reboot", }, }, }, From 59ba3e6892166becf1b7563e30e3cce64f0fae07 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 27 Jul 2023 13:51:34 +0200 Subject: [PATCH 10/50] Update crds --- charts/crds/templates/crds.yaml | 24 +++++++++++++++++++ ...mental.cattle.io_machineregistrations.yaml | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/charts/crds/templates/crds.yaml b/charts/crds/templates/crds.yaml index 00e7ece74..636756de9 100644 --- a/charts/crds/templates/crds.yaml +++ b/charts/crds/templates/crds.yaml @@ -705,6 +705,30 @@ spec: url: type: string type: object + reset: + properties: + config-urls: + items: + type: string + type: array + debug: + type: boolean + enabled: + type: boolean + poweroff: + type: boolean + reboot: + default: true + type: boolean + reset-oem: + default: true + type: boolean + reset-persistent: + default: true + type: boolean + system-uri: + type: string + type: object system-agent: properties: secret-name: diff --git a/config/crd/bases/elemental.cattle.io_machineregistrations.yaml b/config/crd/bases/elemental.cattle.io_machineregistrations.yaml index eec5132cc..1ec0da7a6 100644 --- a/config/crd/bases/elemental.cattle.io_machineregistrations.yaml +++ b/config/crd/bases/elemental.cattle.io_machineregistrations.yaml @@ -89,6 +89,30 @@ spec: url: type: string type: object + reset: + properties: + config-urls: + items: + type: string + type: array + debug: + type: boolean + enabled: + type: boolean + poweroff: + type: boolean + reboot: + default: true + type: boolean + reset-oem: + default: true + type: boolean + reset-persistent: + default: true + type: boolean + system-uri: + type: string + type: object system-agent: properties: secret-name: From 8b8798f790cfc2e80ca6305b4d55fe9856e0b21b Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 27 Jul 2023 14:51:40 +0200 Subject: [PATCH 11/50] Use mInventory name and namespace to determine elemental-system-agent watched secret --- controllers/machineinventory_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 908b1036e..7bee4cdb0 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -220,8 +220,8 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte planSecret := &corev1.Secret{} if err := r.Get(ctx, types.NamespacedName{ - Namespace: mInventory.Status.Plan.PlanSecretRef.Namespace, - Name: mInventory.Status.Plan.PlanSecretRef.Name, + Namespace: mInventory.Namespace, + Name: mInventory.Name, }, planSecret); err != nil { return fmt.Errorf("getting plan secret: %w", err) } From da296fe3ce0284693b45eb9fecf826c50ae34e0c Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 15:16:58 +0200 Subject: [PATCH 12/50] Fix incorrect handling of registration state within a live environment --- cmd/register/main.go | 36 ++++----- cmd/register/main_test.go | 88 +++++++++++++++------- controllers/machineinventory_controller.go | 2 +- pkg/install/install.go | 59 +++++++++++++-- pkg/install/mocks/installer.go | 9 ++- pkg/register/mocks/client.go | 9 ++- pkg/register/register.go | 25 ++---- 7 files changed, 149 insertions(+), 79 deletions(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index 9a464bfd1..a5584a43e 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -36,6 +36,7 @@ import ( const ( defaultStatePath = "/oem/registration/state.yaml" + defaultLiveStatePath = "/tmp/registration/state.yaml" defaultConfigPath = "/oem/registration/config.yaml" defaultLiveConfigPath = "/run/initramfs/live/livecd-cloud-config.yaml" registrationUpdateSuppressTimer = 24 * time.Hour @@ -84,17 +85,18 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err := initConfig(fs); err != nil { return fmt.Errorf("initializing configuration: %w", err) } + // Load Registration State + if err := stateHandler.Init(statePath); err != nil { + return fmt.Errorf("initializing state handler on path '%s': %w", statePath, err) + } + registrationState, err := stateHandler.Load() + if err != nil { + return fmt.Errorf("loading registration state: %w", err) + } // Determine if registration should execute or skip a cycle - if !installation && !reset { - if err := stateHandler.Init(statePath); err != nil { - return fmt.Errorf("initializing state handler on path '%s': %w", statePath, err) - } - if skip, err := shouldSkipRegistration(stateHandler); err != nil { - return fmt.Errorf("determining if registration should run: %w", err) - } else if skip { - log.Info("Nothing to do") - return nil - } + if !installation && !reset && !registrationState.HasLastUpdateElapsed(registrationUpdateSuppressTimer) { + log.Info("Nothing to do") + return nil } // Validate CA caCert, err := getRegistrationCA(fs, cfg) @@ -102,10 +104,11 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa return fmt.Errorf("validating CA: %w", err) } // Register (and fetch the remote MachineRegistration) - data, err := client.Register(cfg.Elemental.Registration, caCert) + data, err := client.Register(cfg.Elemental.Registration, caCert, ®istrationState) if err != nil { return fmt.Errorf("registering machine: %w", err) } + stateHandler.Save(registrationState) // Validate remote config log.Debugf("Fetched configuration from manager cluster:\n%s\n\n", string(data)) if err := yaml.Unmarshal(data, &cfg); err != nil { @@ -114,7 +117,9 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa // Install if installation { log.Info("Installing elemental") - return installer.InstallElemental(cfg) + if err := installer.InstallElemental(cfg, registrationState); err != nil { + return fmt.Errorf("installing elemental: %w", err) + } } // Reset if reset { @@ -159,6 +164,7 @@ func initConfig(fs vfs.FS) error { // If we are installing from a live environment, the default config path must be updated if installation && (configPath == defaultConfigPath) { configPath = defaultLiveConfigPath + statePath = defaultLiveStatePath } // Use go-vfs afero compatibility layer (required by Viper) afs := vfsafero.NewAferoFS(fs) @@ -176,11 +182,7 @@ func initConfig(fs vfs.FS) error { return nil } -func shouldSkipRegistration(stateHandler register.StateHandler) (bool, error) { - state, err := stateHandler.Load() - if err != nil { - return false, fmt.Errorf("loading registration state") - } +func shouldSkipRegistration(state register.State) (bool, error) { return !state.HasLastUpdateElapsed(registrationUpdateSuppressTimer), nil } diff --git a/cmd/register/main_test.go b/cmd/register/main_test.go index 89c8e3e60..fc4855cec 100644 --- a/cmd/register/main_test.go +++ b/cmd/register/main_test.go @@ -90,6 +90,12 @@ var ( }, }, } + stateFixture = register.State{ + InitialRegistration: time.Now(), + LastUpdate: time.Time{}, + EmulatedTPM: true, + EmulatedTPMSeed: 987654321, + } ) var _ = Describe("elemental-register", Label("registration", "cli"), func() { @@ -100,13 +106,15 @@ var _ = Describe("elemental-register", Label("registration", "cli"), func() { var mockCtrl *gomock.Controller var client *rmocks.MockClient var installer *imocks.MockInstaller + var stateHandler *rmocks.MockStateHandler BeforeEach(func() { fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) Expect(err).ToNot(HaveOccurred()) mockCtrl = gomock.NewController(GinkgoT()) client = rmocks.NewMockClient(mockCtrl) installer = imocks.NewMockInstaller(mockCtrl) - cmd = newCommand(fs, client, register.NewFileStateHandler(fs), installer) + stateHandler = rmocks.NewMockStateHandler(mockCtrl) + cmd = newCommand(fs, client, stateHandler, installer) DeferCleanup(fsCleanup) }) It("should return no error when printing version", func() { @@ -116,11 +124,14 @@ var _ = Describe("elemental-register", Label("registration", "cli"), func() { When("using existing default config", func() { BeforeEach(func() { marshalIntoFile(fs, baseConfigFixture, defaultConfigPath) + stateHandler.EXPECT().Init(defaultStatePath).Return(nil) + stateHandler.EXPECT().Load().Return(stateFixture, nil) + stateHandler.EXPECT().Save(stateFixture).Return(nil) }) It("should use the config if no arguments passed", func() { cmd.SetArgs([]string{}) client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert), &stateFixture). Return(marshalToBytes(baseConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) @@ -136,7 +147,7 @@ var _ = Describe("elemental-register", Label("registration", "cli"), func() { wantConfig := alternateConfigFixture.DeepCopy() wantConfig.Elemental.Registration.NoSMBIOS = false client.EXPECT(). - Register(wantConfig.Elemental.Registration, []byte(wantConfig.Elemental.Registration.CACert)). + Register(wantConfig.Elemental.Registration, []byte(wantConfig.Elemental.Registration.CACert), &stateFixture). Return(marshalToBytes(wantConfig), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) @@ -145,18 +156,46 @@ var _ = Describe("elemental-register", Label("registration", "cli"), func() { cmd.SetArgs([]string{"--config-path", newPath}) marshalIntoFile(fs, alternateConfigFixture, newPath) client.EXPECT(). - Register(alternateConfigFixture.Elemental.Registration, []byte(alternateConfigFixture.Elemental.Registration.CACert)). + Register(alternateConfigFixture.Elemental.Registration, []byte(alternateConfigFixture.Elemental.Registration.CACert), &stateFixture). Return(marshalToBytes(alternateConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) - It("should skip registration if lastUpdate is recent", func() { - cmd.SetArgs([]string{}) + }) +}) + +var _ = Describe("elemental-register state", Label("registration", "cli", "state"), func() { + var fs vfs.FS + var err error + var fsCleanup func() + var cmd *cobra.Command + var mockCtrl *gomock.Controller + var client *rmocks.MockClient + var installer *imocks.MockInstaller + var stateHandler *rmocks.MockStateHandler + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + mockCtrl = gomock.NewController(GinkgoT()) + client = rmocks.NewMockClient(mockCtrl) + installer = imocks.NewMockInstaller(mockCtrl) + stateHandler = rmocks.NewMockStateHandler(mockCtrl) + cmd = newCommand(fs, client, stateHandler, installer) + DeferCleanup(fsCleanup) + }) + When("using existing default config", func() { + BeforeEach(func() { + marshalIntoFile(fs, baseConfigFixture, defaultConfigPath) + }) + It("should use state path argument", func() { + newPath := "/a/custom/state/path/custom-state.yaml" + cmd.SetArgs([]string{"--state-path", newPath}) registrationState := register.State{ InitialRegistration: time.Now(), LastUpdate: time.Now(), } - marshalIntoFile(fs, registrationState, defaultStatePath) - client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) + stateHandler.EXPECT().Init(newPath).Return(nil) + stateHandler.EXPECT().Load().Return(registrationState, nil) + client.EXPECT().Register(gomock.Any(), gomock.Any(), gomock.Any()).Times(0) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) It("should not skip registration if lastUpdate is stale", func() { @@ -165,23 +204,14 @@ var _ = Describe("elemental-register", Label("registration", "cli"), func() { InitialRegistration: time.Now(), LastUpdate: time.Now().Add(-25 * time.Hour), } - marshalIntoFile(fs, registrationState, defaultStatePath) + stateHandler.EXPECT().Init(defaultStatePath).Return(nil) + stateHandler.EXPECT().Load().Return(registrationState, nil) + stateHandler.EXPECT().Save(registrationState).Return(nil) client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert), ®istrationState). Return(marshalToBytes(baseConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) - It("should use state path argument", func() { - newPath := "/a/custom/state/path/custom-state.yaml" - cmd.SetArgs([]string{"--state-path", newPath}) - registrationState := register.State{ - InitialRegistration: time.Now(), - LastUpdate: time.Now(), - } - marshalIntoFile(fs, registrationState, newPath) - client.EXPECT().Register(gomock.Any(), gomock.Any()).Times(0) - Expect(cmd.Execute()).ToNot(HaveOccurred()) - }) }) }) @@ -199,7 +229,7 @@ var _ = Describe("elemental-register --install", Label("registration", "cli", "i Expect(err).ToNot(HaveOccurred()) mockCtrl = gomock.NewController(GinkgoT()) installer = imocks.NewMockInstaller(mockCtrl) - stateHandler = rmocks.NewMockStateHandler(mockCtrl) // Expect no calls + stateHandler = rmocks.NewMockStateHandler(mockCtrl) client = rmocks.NewMockClient(mockCtrl) cmd = newCommand(fs, client, stateHandler, installer) DeferCleanup(fsCleanup) @@ -207,12 +237,15 @@ var _ = Describe("elemental-register --install", Label("registration", "cli", "i When("using existing live config", func() { BeforeEach(func() { marshalIntoFile(fs, baseConfigFixture, defaultLiveConfigPath) + stateHandler.EXPECT().Init(defaultLiveStatePath).Return(nil) + stateHandler.EXPECT().Load().Return(stateFixture, nil) + stateHandler.EXPECT().Save(stateFixture).Return(nil) }) It("should trigger install when --install argument", func() { cmd.SetArgs([]string{"--install"}) - installer.EXPECT().InstallElemental(alternateConfigFixture).Return(nil) + installer.EXPECT().InstallElemental(alternateConfigFixture, stateFixture).Return(nil) client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert), &stateFixture). Return(marshalToBytes(alternateConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) @@ -233,7 +266,7 @@ var _ = Describe("elemental-register --reset", Label("registration", "cli", "res Expect(err).ToNot(HaveOccurred()) mockCtrl = gomock.NewController(GinkgoT()) installer = imocks.NewMockInstaller(mockCtrl) - stateHandler = rmocks.NewMockStateHandler(mockCtrl) // Expect no calls + stateHandler = rmocks.NewMockStateHandler(mockCtrl) client = rmocks.NewMockClient(mockCtrl) cmd = newCommand(fs, client, stateHandler, installer) DeferCleanup(fsCleanup) @@ -241,12 +274,15 @@ var _ = Describe("elemental-register --reset", Label("registration", "cli", "res When("using existing default config", func() { BeforeEach(func() { marshalIntoFile(fs, baseConfigFixture, defaultConfigPath) + stateHandler.EXPECT().Init(defaultStatePath).Return(nil) + stateHandler.EXPECT().Load().Return(stateFixture, nil) + stateHandler.EXPECT().Save(stateFixture).Return(nil) }) It("should trigger reset when --reset argument", func() { cmd.SetArgs([]string{"--reset"}) installer.EXPECT().ResetElemental(alternateConfigFixture).Return(nil) client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert)). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert), &stateFixture). Return(marshalToBytes(alternateConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 7bee4cdb0..6cab30c5a 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -261,7 +261,7 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, Stages: map[string][]schema.Stage{ "network": { schema.Stage{ - If: "'[ -f /run/cos/recovery_mode ]'", + If: "[ -f /run/cos/recovery_mode ]", Name: "Runs elemental reset", Commands: []string{ "elemental-register --debug --reset", diff --git a/pkg/install/install.go b/pkg/install/install.go index eb0748de7..1f5c1fecc 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -25,6 +25,7 @@ import ( elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" "github.com/rancher/elemental-operator/pkg/elementalcli" "github.com/rancher/elemental-operator/pkg/log" + "github.com/rancher/elemental-operator/pkg/register" "github.com/rancher/elemental-operator/pkg/util" agent "github.com/rancher/system-agent/pkg/config" "github.com/twpayne/go-vfs" @@ -35,15 +36,16 @@ import ( ) const ( - stateInstallFile = "/run/initramfs/cos-state/state.yaml" - agentStateDir = "/var/lib/elemental/agent" - agentConfDir = "/etc/rancher/elemental/agent" - registrationConf = "/oem/registration/config.yaml" + stateInstallFile = "/run/initramfs/cos-state/state.yaml" + agentStateDir = "/var/lib/elemental/agent" + agentConfDir = "/etc/rancher/elemental/agent" + registrationConf = "/oem/registration/config.yaml" + registrationState = "/oem/registration/state.yaml" ) type Installer interface { ResetElemental(config elementalv1.Config) error - InstallElemental(config elementalv1.Config) error + InstallElemental(config elementalv1.Config, state register.State) error } func NewInstaller(fs vfs.FS) Installer { @@ -60,7 +62,7 @@ type installer struct { runner elementalcli.Runner } -func (i *installer) InstallElemental(config elementalv1.Config) error { +func (i *installer) InstallElemental(config elementalv1.Config, state register.State) error { if config.Elemental.Install.ConfigURLs == nil { config.Elemental.Install.ConfigURLs = []string{} } @@ -69,6 +71,13 @@ func (i *installer) InstallElemental(config elementalv1.Config) error { if err != nil { return fmt.Errorf("generating additional cloud configs: %w", err) } + registrationStatePath, err := i.writeRegistrationState(state) + + if err != nil { + return fmt.Errorf("writing registration state plan: %w", err) + } + additionalConfigs = append(additionalConfigs, registrationStatePath) + config.Elemental.Install.ConfigURLs = append(config.Elemental.Install.ConfigURLs, additionalConfigs...) if err := i.runner.Install(config.Elemental.Install); err != nil { @@ -135,7 +144,7 @@ func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, }, }) if err != nil { - return "", err + return "", fmt.Errorf("marshalling registration config: %w", err) } err = yaml.NewEncoder(f).Encode(schema.YipConfig{ @@ -163,6 +172,42 @@ func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, return f.Name(), err } +func (i *installer) writeRegistrationState(state register.State) (string, error) { + f, err := i.fs.Create("/tmp/elemental-registration-state.yaml") + if err != nil { + return "", fmt.Errorf("creating temporary registration state plan file: %w", err) + } + defer f.Close() + stateBytes, err := yaml.Marshal(state) + if err != nil { + return "", fmt.Errorf("marshalling registration state: %w", err) + } + + err = yaml.NewEncoder(f).Encode(schema.YipConfig{ + Name: "Include registration state into installed system", + Stages: map[string][]schema.Stage{ + "initramfs": { + schema.Stage{ + If: fmt.Sprintf("[ ! -f %s ]", registrationState), + Directories: []schema.Directory{ + { + Path: filepath.Dir(registrationState), + Permissions: 0700, + }, + }, Files: []schema.File{ + { + Path: registrationState, + Content: string(stateBytes), + Permissions: 0600, + }, + }, + }, + }, + }, + }) + return f.Name(), err +} + func (i *installer) writeCloudInit(cloudConfig map[string]runtime.RawExtension) (string, error) { f, err := i.fs.Create("/tmp/elemental-cloud-init.yaml") if err != nil { diff --git a/pkg/install/mocks/installer.go b/pkg/install/mocks/installer.go index 98bd6d113..a6c1ff107 100644 --- a/pkg/install/mocks/installer.go +++ b/pkg/install/mocks/installer.go @@ -8,6 +8,7 @@ import ( reflect "reflect" v1beta1 "github.com/rancher/elemental-operator/api/v1beta1" + register "github.com/rancher/elemental-operator/pkg/register" gomock "go.uber.org/mock/gomock" ) @@ -35,17 +36,17 @@ func (m *MockInstaller) EXPECT() *MockInstallerMockRecorder { } // InstallElemental mocks base method. -func (m *MockInstaller) InstallElemental(arg0 v1beta1.Config) error { +func (m *MockInstaller) InstallElemental(arg0 v1beta1.Config, arg1 register.State) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallElemental", arg0) + ret := m.ctrl.Call(m, "InstallElemental", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // InstallElemental indicates an expected call of InstallElemental. -func (mr *MockInstallerMockRecorder) InstallElemental(arg0 interface{}) *gomock.Call { +func (mr *MockInstallerMockRecorder) InstallElemental(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallElemental", reflect.TypeOf((*MockInstaller)(nil).InstallElemental), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallElemental", reflect.TypeOf((*MockInstaller)(nil).InstallElemental), arg0, arg1) } // ResetElemental mocks base method. diff --git a/pkg/register/mocks/client.go b/pkg/register/mocks/client.go index cd9da119c..71bf89086 100644 --- a/pkg/register/mocks/client.go +++ b/pkg/register/mocks/client.go @@ -8,6 +8,7 @@ import ( reflect "reflect" v1beta1 "github.com/rancher/elemental-operator/api/v1beta1" + register "github.com/rancher/elemental-operator/pkg/register" gomock "go.uber.org/mock/gomock" ) @@ -35,16 +36,16 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // Register mocks base method. -func (m *MockClient) Register(arg0 v1beta1.Registration, arg1 []byte) ([]byte, error) { +func (m *MockClient) Register(arg0 v1beta1.Registration, arg1 []byte, arg2 *register.State) ([]byte, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Register", arg0, arg1) + ret := m.ctrl.Call(m, "Register", arg0, arg1, arg2) ret0, _ := ret[0].([]byte) ret1, _ := ret[1].(error) return ret0, ret1 } // Register indicates an expected call of Register. -func (mr *MockClientMockRecorder) Register(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) Register(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockClient)(nil).Register), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockClient)(nil).Register), arg0, arg1, arg2) } diff --git a/pkg/register/register.go b/pkg/register/register.go index d67642af2..a330e531f 100644 --- a/pkg/register/register.go +++ b/pkg/register/register.go @@ -39,7 +39,7 @@ import ( ) type Client interface { - Register(reg elementalv1.Registration, caCert []byte) ([]byte, error) + Register(reg elementalv1.Registration, caCert []byte, state *State) ([]byte, error) } type authClient interface { @@ -52,26 +52,16 @@ type authClient interface { var _ Client = (*client)(nil) -type client struct { - stateHandler StateHandler -} +type client struct{} func NewClient(stateHandler StateHandler) Client { - return &client{ - stateHandler: stateHandler, - } + return &client{} } // Register attempts to register the machine with the elemental-operator. -// If the machine is already installed and registered, a registration can still be attempted turning the `isUpdate` flag on. // Registration updates will fetch and apply new labels, and update Machine annotations such as the IP address. -func (r *client) Register(reg elementalv1.Registration, caCert []byte) ([]byte, error) { - state, err := r.stateHandler.Load() - if err != nil { - return nil, fmt.Errorf("loading registration state: %w", err) - } - - auth, err := getAuthenticator(reg, &state) +func (r *client) Register(reg elementalv1.Registration, caCert []byte, state *State) ([]byte, error) { + auth, err := getAuthenticator(reg, state) if err != nil { return nil, fmt.Errorf("initializing authenticator: %w", err) } @@ -130,11 +120,6 @@ func (r *client) Register(reg elementalv1.Registration, caCert []byte) ([]byte, } } - log.Info("Saving registration state") - if err := r.stateHandler.Save(state); err != nil { - return nil, fmt.Errorf("saving registration state: %w", err) - } - log.Info("Get elemental configuration") if err := WriteMessage(conn, MsgGet, []byte{}); err != nil { return nil, fmt.Errorf("request elemental configuration: %w", err) From 420f57a85f70927c5c29be9fb0cd06bc4b2ad583 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 15:24:59 +0200 Subject: [PATCH 13/50] Make the linter happy --- cmd/register/main.go | 10 ++++------ pkg/register/register.go | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index a5584a43e..402ea7045 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -59,7 +59,7 @@ func main() { fs := vfs.OSFS installer := install.NewInstaller(fs) stateHandler := register.NewFileStateHandler(fs) - client := register.NewClient(stateHandler) + client := register.NewClient() cmd := newCommand(fs, client, stateHandler, installer) if err := cmd.Execute(); err != nil { log.Fatalf("FATAL: %s", err) @@ -108,7 +108,9 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err != nil { return fmt.Errorf("registering machine: %w", err) } - stateHandler.Save(registrationState) + if err := stateHandler.Save(registrationState); err != nil { + return fmt.Errorf("saving registration state: %w", err) + } // Validate remote config log.Debugf("Fetched configuration from manager cluster:\n%s\n\n", string(data)) if err := yaml.Unmarshal(data, &cfg); err != nil { @@ -182,10 +184,6 @@ func initConfig(fs vfs.FS) error { return nil } -func shouldSkipRegistration(state register.State) (bool, error) { - return !state.HasLastUpdateElapsed(registrationUpdateSuppressTimer), nil -} - func getRegistrationCA(fs vfs.FS, config elementalv1.Config) ([]byte, error) { registration := config.Elemental.Registration diff --git a/pkg/register/register.go b/pkg/register/register.go index a330e531f..33a93742e 100644 --- a/pkg/register/register.go +++ b/pkg/register/register.go @@ -54,7 +54,7 @@ var _ Client = (*client)(nil) type client struct{} -func NewClient(stateHandler StateHandler) Client { +func NewClient() Client { return &client{} } From 40bdf1d221112ffff3f6f5ad1a3dca07705a0a2e Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 17:17:28 +0200 Subject: [PATCH 14/50] Fix local reset plan location and schema --- controllers/machineinventory_controller.go | 78 +++++++++++++--------- pkg/install/install.go | 6 +- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 6cab30c5a..f52f1c57b 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -25,7 +25,6 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/mudler/yip/pkg/schema" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -71,13 +70,9 @@ func (r *MachineInventoryReconciler) Reconcile(ctx context.Context, req reconcil logger := ctrl.LoggerFrom(ctx) mInventory := &elementalv1.MachineInventory{} - err := r.Get(ctx, req.NamespacedName, mInventory) - if err != nil { - if apierrors.IsNotFound(err) { - logger.V(log.DebugDepth).Info("Object was not found, not an error") - return reconcile.Result{}, nil - } - return reconcile.Result{}, fmt.Errorf("failed to get machine inventory object: %w", err) + if err := r.Get(ctx, req.NamespacedName, mInventory); err != nil { + logger.V(log.DebugDepth).Error(err, "Unable to fetch MachineInventory") + return reconcile.Result{}, client.IgnoreNotFound(err) } patchBase := client.MergeFrom(mInventory.DeepCopy()) @@ -117,15 +112,26 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * logger.Info("Reconciling machineinventory object") - if mInventory.GetDeletionTimestamp() != nil { - if err := r.reconcileResetPlanSecret(ctx, mInventory); err != nil { - meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ - Type: elementalv1.ReadyCondition, - Reason: elementalv1.PlanFailureReason, - Status: metav1.ConditionFalse, - Message: err.Error(), - }) - return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) + if mInventory.GetDeletionTimestamp().IsZero() { + // The object is not being deleted, so register the finalizer + if !controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { + controllerutil.AddFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) + if err := r.Update(ctx, mInventory); err != nil { + return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) + } + } + } else { + // The object is up for deletion + if controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { + if err := r.reconcileResetPlanSecret(ctx, mInventory); err != nil { + meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ + Type: elementalv1.ReadyCondition, + Reason: elementalv1.PlanFailureReason, + Status: metav1.ConditionFalse, + Message: err.Error(), + }) + return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) + } } return ctrl.Result{}, nil } @@ -233,6 +239,8 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte patchBase := client.MergeFrom(planSecret.DeepCopy()) + planSecret.Data["applied-checksum"] = []byte("") + planSecret.Data["failed-checksum"] = []byte("") planSecret.Data["plan"] = resetPlan planSecret.Annotations = map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeReset} @@ -255,23 +263,22 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, logger.Info("Creating new Reset plan secret") - // This is the local cloud-config that the elemental-system-agent will run while in recovery mode - resetCloudConfig := schema.YipConfig{ - Name: "Elemental Reset", - Stages: map[string][]schema.Stage{ - "network": { - schema.Stage{ - If: "[ -f /run/cos/recovery_mode ]", - Name: "Runs elemental reset", - Commands: []string{ - "elemental-register --debug --reset", + // This is the local reset plan that the elemental-system-agent will run while in recovery mode + localResetPlan := applyinator.Plan{ + OneTimeInstructions: []applyinator.OneTimeInstruction{ + { + CommonInstruction: applyinator.CommonInstruction{ + Name: "reset elemental", + Command: "elemental-register", + Args: []string{ + "--reset", }, }, }, }, } - resetCloudConfigBytes, err := yaml.Marshal(resetCloudConfig) + localResetPlanBytes, err := yaml.Marshal(localResetPlan) if err != nil { return nil, fmt.Errorf("marshalling local reset cloud-config to yaml: %w", err) } @@ -280,8 +287,13 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, resetPlan := applyinator.Plan{ Files: []applyinator.File{ { - Content: base64.StdEncoding.EncodeToString(resetCloudConfigBytes), - Path: "/oem/reset-plan.yaml", + Directory: true, + Path: "/var/ib/elemental/agent/plans", + Permissions: "0700", + }, + { + Content: base64.StdEncoding.EncodeToString(localResetPlanBytes), + Path: "/var/ib/elemental/agent/plans/reset.plan", Permissions: "0600", }, }, @@ -300,7 +312,11 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, { CommonInstruction: applyinator.CommonInstruction{ Name: "reboot", - Command: "shutdown -r +1", // Need to have time to confirm plan execution before rebooting + Command: "shutdown", + Args: []string{ + "-r", + "+1", // Need to have time to confirm plan execution before rebooting + }, }, }, }, diff --git a/pkg/install/install.go b/pkg/install/install.go index 1f5c1fecc..955a25423 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -157,7 +157,8 @@ func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, Path: filepath.Dir(registrationConf), Permissions: 0700, }, - }, Files: []schema.File{ + }, + Files: []schema.File{ { Path: registrationConf, Content: string(registrationInBytes), @@ -194,7 +195,8 @@ func (i *installer) writeRegistrationState(state register.State) (string, error) Path: filepath.Dir(registrationState), Permissions: 0700, }, - }, Files: []schema.File{ + }, + Files: []schema.File{ { Path: registrationState, Content: string(stateBytes), From 48ab712271ba63aa14fc2c6f414fd37b507f38b0 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 17:19:39 +0200 Subject: [PATCH 15/50] Add debug logging to --reset --- controllers/machineinventory_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index f52f1c57b..c45fd5fb4 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -271,6 +271,7 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, Name: "reset elemental", Command: "elemental-register", Args: []string{ + "--debug", "--reset", }, }, From 571f6d70954b61eb84b0dc2b1adb5262c5b12f65 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 17:26:55 +0200 Subject: [PATCH 16/50] Typo fix --- controllers/machineinventory_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index c45fd5fb4..b477c44d0 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -289,12 +289,12 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, Files: []applyinator.File{ { Directory: true, - Path: "/var/ib/elemental/agent/plans", + Path: "/var/lib/elemental/agent/plans", Permissions: "0700", }, { Content: base64.StdEncoding.EncodeToString(localResetPlanBytes), - Path: "/var/ib/elemental/agent/plans/reset.plan", + Path: "/var/lib/elemental/agent/plans/reset.plan", Permissions: "0600", }, }, From 1437d51286bd321822f39550def809f48d5d10fd Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 17:39:49 +0200 Subject: [PATCH 17/50] Revert "Typo fix" This reverts commit 571f6d70954b61eb84b0dc2b1adb5262c5b12f65. --- controllers/machineinventory_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index b477c44d0..c45fd5fb4 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -289,12 +289,12 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, Files: []applyinator.File{ { Directory: true, - Path: "/var/lib/elemental/agent/plans", + Path: "/var/ib/elemental/agent/plans", Permissions: "0700", }, { Content: base64.StdEncoding.EncodeToString(localResetPlanBytes), - Path: "/var/lib/elemental/agent/plans/reset.plan", + Path: "/var/ib/elemental/agent/plans/reset.plan", Permissions: "0600", }, }, From 68312344abd3c09869ad040d0131f45a4f675e55 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 17:40:16 +0200 Subject: [PATCH 18/50] Revert "Add debug logging to --reset" This reverts commit 48ab712271ba63aa14fc2c6f414fd37b507f38b0. --- controllers/machineinventory_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index c45fd5fb4..f52f1c57b 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -271,7 +271,6 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, Name: "reset elemental", Command: "elemental-register", Args: []string{ - "--debug", "--reset", }, }, From 0da25fe0c467832d359d4895fd46912b91e12500 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 17:40:35 +0200 Subject: [PATCH 19/50] Revert "Fix local reset plan location and schema" This reverts commit 40bdf1d221112ffff3f6f5ad1a3dca07705a0a2e. --- controllers/machineinventory_controller.go | 78 +++++++++------------- pkg/install/install.go | 6 +- 2 files changed, 33 insertions(+), 51 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index f52f1c57b..6cab30c5a 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -25,6 +25,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/mudler/yip/pkg/schema" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -70,9 +71,13 @@ func (r *MachineInventoryReconciler) Reconcile(ctx context.Context, req reconcil logger := ctrl.LoggerFrom(ctx) mInventory := &elementalv1.MachineInventory{} - if err := r.Get(ctx, req.NamespacedName, mInventory); err != nil { - logger.V(log.DebugDepth).Error(err, "Unable to fetch MachineInventory") - return reconcile.Result{}, client.IgnoreNotFound(err) + err := r.Get(ctx, req.NamespacedName, mInventory) + if err != nil { + if apierrors.IsNotFound(err) { + logger.V(log.DebugDepth).Info("Object was not found, not an error") + return reconcile.Result{}, nil + } + return reconcile.Result{}, fmt.Errorf("failed to get machine inventory object: %w", err) } patchBase := client.MergeFrom(mInventory.DeepCopy()) @@ -112,26 +117,15 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * logger.Info("Reconciling machineinventory object") - if mInventory.GetDeletionTimestamp().IsZero() { - // The object is not being deleted, so register the finalizer - if !controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { - controllerutil.AddFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) - if err := r.Update(ctx, mInventory); err != nil { - return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) - } - } - } else { - // The object is up for deletion - if controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { - if err := r.reconcileResetPlanSecret(ctx, mInventory); err != nil { - meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ - Type: elementalv1.ReadyCondition, - Reason: elementalv1.PlanFailureReason, - Status: metav1.ConditionFalse, - Message: err.Error(), - }) - return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) - } + if mInventory.GetDeletionTimestamp() != nil { + if err := r.reconcileResetPlanSecret(ctx, mInventory); err != nil { + meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ + Type: elementalv1.ReadyCondition, + Reason: elementalv1.PlanFailureReason, + Status: metav1.ConditionFalse, + Message: err.Error(), + }) + return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) } return ctrl.Result{}, nil } @@ -239,8 +233,6 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte patchBase := client.MergeFrom(planSecret.DeepCopy()) - planSecret.Data["applied-checksum"] = []byte("") - planSecret.Data["failed-checksum"] = []byte("") planSecret.Data["plan"] = resetPlan planSecret.Annotations = map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeReset} @@ -263,22 +255,23 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, logger.Info("Creating new Reset plan secret") - // This is the local reset plan that the elemental-system-agent will run while in recovery mode - localResetPlan := applyinator.Plan{ - OneTimeInstructions: []applyinator.OneTimeInstruction{ - { - CommonInstruction: applyinator.CommonInstruction{ - Name: "reset elemental", - Command: "elemental-register", - Args: []string{ - "--reset", + // This is the local cloud-config that the elemental-system-agent will run while in recovery mode + resetCloudConfig := schema.YipConfig{ + Name: "Elemental Reset", + Stages: map[string][]schema.Stage{ + "network": { + schema.Stage{ + If: "[ -f /run/cos/recovery_mode ]", + Name: "Runs elemental reset", + Commands: []string{ + "elemental-register --debug --reset", }, }, }, }, } - localResetPlanBytes, err := yaml.Marshal(localResetPlan) + resetCloudConfigBytes, err := yaml.Marshal(resetCloudConfig) if err != nil { return nil, fmt.Errorf("marshalling local reset cloud-config to yaml: %w", err) } @@ -287,13 +280,8 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, resetPlan := applyinator.Plan{ Files: []applyinator.File{ { - Directory: true, - Path: "/var/ib/elemental/agent/plans", - Permissions: "0700", - }, - { - Content: base64.StdEncoding.EncodeToString(localResetPlanBytes), - Path: "/var/ib/elemental/agent/plans/reset.plan", + Content: base64.StdEncoding.EncodeToString(resetCloudConfigBytes), + Path: "/oem/reset-plan.yaml", Permissions: "0600", }, }, @@ -312,11 +300,7 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, { CommonInstruction: applyinator.CommonInstruction{ Name: "reboot", - Command: "shutdown", - Args: []string{ - "-r", - "+1", // Need to have time to confirm plan execution before rebooting - }, + Command: "shutdown -r +1", // Need to have time to confirm plan execution before rebooting }, }, }, diff --git a/pkg/install/install.go b/pkg/install/install.go index 955a25423..1f5c1fecc 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -157,8 +157,7 @@ func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, Path: filepath.Dir(registrationConf), Permissions: 0700, }, - }, - Files: []schema.File{ + }, Files: []schema.File{ { Path: registrationConf, Content: string(registrationInBytes), @@ -195,8 +194,7 @@ func (i *installer) writeRegistrationState(state register.State) (string, error) Path: filepath.Dir(registrationState), Permissions: 0700, }, - }, - Files: []schema.File{ + }, Files: []schema.File{ { Path: registrationState, Content: string(stateBytes), From 65a9ca9f697e6e64368a39139596a6eb36591784 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Fri, 28 Jul 2023 17:41:58 +0200 Subject: [PATCH 20/50] Fix shutdown command arguments --- controllers/machineinventory_controller.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 6cab30c5a..cf7d0bf56 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -299,8 +299,12 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, }, { CommonInstruction: applyinator.CommonInstruction{ - Name: "reboot", - Command: "shutdown -r +1", // Need to have time to confirm plan execution before rebooting + Name: "schedule reboot", + Command: "shutdown", + Args: []string{ + "-r", + "+1", // Need to have time to confirm plan execution before rebooting + }, }, }, }, From c1cb54b9c71ab20c8d0f22cdb4b47bf9b3592f5e Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 09:44:12 +0200 Subject: [PATCH 21/50] Blank previos plan status --- controllers/machineinventory_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index cf7d0bf56..4235d758c 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -233,6 +233,8 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte patchBase := client.MergeFrom(planSecret.DeepCopy()) + planSecret.Data["applied-checksum"] = []byte("") + planSecret.Data["failed-checksum"] = []byte("") planSecret.Data["plan"] = resetPlan planSecret.Annotations = map[string]string{elementalv1.PlanTypeAnnotation: elementalv1.PlanTypeReset} From 339445db00169344b4abccf94ebf9376c65a91cf Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 09:51:19 +0200 Subject: [PATCH 22/50] Apply Finalizer on MachineInventory --- controllers/machineinventory_controller.go | 29 +++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 4235d758c..68178a858 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -117,15 +117,26 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * logger.Info("Reconciling machineinventory object") - if mInventory.GetDeletionTimestamp() != nil { - if err := r.reconcileResetPlanSecret(ctx, mInventory); err != nil { - meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ - Type: elementalv1.ReadyCondition, - Reason: elementalv1.PlanFailureReason, - Status: metav1.ConditionFalse, - Message: err.Error(), - }) - return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) + if mInventory.GetDeletionTimestamp().IsZero() { + // The object is not being deleted, so register the finalizer + if !controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { + controllerutil.AddFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) + if err := r.Update(ctx, mInventory); err != nil { + return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) + } + } + } else { + // The object is up for deletion + if controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { + if err := r.reconcileResetPlanSecret(ctx, mInventory); err != nil { + meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ + Type: elementalv1.ReadyCondition, + Reason: elementalv1.PlanFailureReason, + Status: metav1.ConditionFalse, + Message: err.Error(), + }) + return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) + } } return ctrl.Result{}, nil } From 3cc844eb84491e21ffbcfe06d4df35d7c6ed9c98 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 11:28:46 +0200 Subject: [PATCH 23/50] Reset mInventory plan status on new plan created --- controllers/machineinventory_controller.go | 19 +++++++++++++------ controllers/machineselector_controller.go | 10 +--------- pkg/util/util.go | 8 ++++++++ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 68178a858..68cf9ebd1 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -212,7 +212,7 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex return r.updatePlanSecretWithReset(ctx, mInventory) } - logger.V(log.DebugDepth).Info("Reset plan type found. Updating status to determine whether it was successfully applied.") + logger.V(log.DebugDepth).Info("Reset plan type found. Updating status and determine whether it was successfully applied.") if err := r.updateInventoryWithPlanStatus(ctx, mInventory); err != nil { return fmt.Errorf("updating inventory with plan status: %w", err) } @@ -237,7 +237,7 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte return fmt.Errorf("getting plan secret: %w", err) } - resetPlan, err := r.newResetPlan(ctx) + checksum, resetPlan, err := r.newResetPlan(ctx) if err != nil { return fmt.Errorf("getting new reset plan: %w", err) } @@ -253,6 +253,11 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte return fmt.Errorf("patching plan secret: %w", err) } + mInventory.Status.Plan.Checksum = checksum + mInventory.Status.Plan.PlanSecretRef.Name = planSecret.Name + mInventory.Status.Plan.PlanSecretRef.Namespace = planSecret.Namespace + mInventory.Status.Plan.State = elementalv1.PlanState("") + meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ Type: elementalv1.ReadyCondition, Reason: elementalv1.WaitingForPlanReason, @@ -263,7 +268,7 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte return nil } -func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, error) { +func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) (string, []byte, error) { logger := ctrl.LoggerFrom(ctx) logger.Info("Creating new Reset plan secret") @@ -286,7 +291,7 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, resetCloudConfigBytes, err := yaml.Marshal(resetCloudConfig) if err != nil { - return nil, fmt.Errorf("marshalling local reset cloud-config to yaml: %w", err) + return "", nil, fmt.Errorf("marshalling local reset cloud-config to yaml: %w", err) } // This is the remote plan that should trigger the reboot into recovery and reset @@ -325,12 +330,14 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) ([]byte, var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(resetPlan); err != nil { - return nil, fmt.Errorf("failed to encode plan: %w", err) + return "", nil, fmt.Errorf("failed to encode plan: %w", err) } plan := buf.Bytes() - return plan, nil + checksum := util.PlanChecksum(plan) + + return checksum, plan, nil } func (r *MachineInventoryReconciler) createPlanSecret(ctx context.Context, mInventory *elementalv1.MachineInventory) error { diff --git a/controllers/machineselector_controller.go b/controllers/machineselector_controller.go index df189681d..2f39030af 100644 --- a/controllers/machineselector_controller.go +++ b/controllers/machineselector_controller.go @@ -19,7 +19,6 @@ package controllers import ( "bytes" "context" - "crypto/sha256" "fmt" "time" @@ -485,7 +484,7 @@ func (r *MachineInventorySelectorReconciler) newBootstrapPlan(ctx context.Contex plan := buf.Bytes() - checksum := planChecksum(plan) + checksum := util.PlanChecksum(plan) return checksum, plan, nil } @@ -602,13 +601,6 @@ func filterSelectorUpdateEvents() predicate.Funcs { } } -func planChecksum(input []byte) string { - h := sha256.New() - h.Write(input) - - return fmt.Sprintf("%x", h.Sum(nil)) -} - // MachineInventoryToSelector is a handler.ToRequestsFunc to be used to enqueue requests for reconciliation // for MachineInventoryToSelector that might adopt a MachineInventory. func (r *MachineInventorySelectorReconciler) MachineInventoryToSelector(o client.Object) []reconcile.Request { diff --git a/pkg/util/util.go b/pkg/util/util.go index 4c1ca2e54..815d72ee6 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -18,6 +18,7 @@ package util import ( "context" + "crypto/sha256" "encoding/json" "fmt" "net/url" @@ -120,3 +121,10 @@ func IsHTTP(uri string) bool { return strings.HasPrefix(parsed.Scheme, "http") } + +func PlanChecksum(input []byte) string { + h := sha256.New() + h.Write(input) + + return fmt.Sprintf("%x", h.Sum(nil)) +} From 6ddf7f75886a5546f6060f5a37ba3ed9ba328a75 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 11:48:12 +0200 Subject: [PATCH 24/50] Fix reset of status plan --- controllers/machineinventory_controller.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 68cf9ebd1..f5c06b67a 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -253,10 +253,13 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte return fmt.Errorf("patching plan secret: %w", err) } - mInventory.Status.Plan.Checksum = checksum - mInventory.Status.Plan.PlanSecretRef.Name = planSecret.Name - mInventory.Status.Plan.PlanSecretRef.Namespace = planSecret.Namespace - mInventory.Status.Plan.State = elementalv1.PlanState("") + mInventory.Status.Plan = &elementalv1.PlanStatus{ + Checksum: checksum, + PlanSecretRef: &corev1.ObjectReference{ + Namespace: planSecret.Namespace, + Name: planSecret.Name, + }, + } meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ Type: elementalv1.ReadyCondition, From acf2cc59e149a6cf81483d09a1ea06688172483c Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 12:03:01 +0200 Subject: [PATCH 25/50] Requeue after adding finalizer --- controllers/machineinventory_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index f5c06b67a..dec97a987 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -124,6 +124,7 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * if err := r.Update(ctx, mInventory); err != nil { return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) } + return ctrl.Result{RequeueAfter: time.Second}, nil } } else { // The object is up for deletion From fb4da6d8dc466ef7c8efac6423718f314de05db4 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 13:24:31 +0200 Subject: [PATCH 26/50] Fix incorrect yaml encoder usage --- controllers/machineinventory_controller.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index dec97a987..ab87793f5 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -39,7 +39,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/yaml" + + "gopkg.in/yaml.v3" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" "github.com/rancher/elemental-operator/pkg/log" @@ -281,7 +282,7 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) (string, resetCloudConfig := schema.YipConfig{ Name: "Elemental Reset", Stages: map[string][]schema.Stage{ - "network": { + "network.after": { schema.Stage{ If: "[ -f /run/cos/recovery_mode ]", Name: "Runs elemental reset", From 225f4ef657268380af627cba99b5c58b8272158f Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 13:24:56 +0200 Subject: [PATCH 27/50] Upgrade yaml encoder version --- pkg/install/install.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/install/install.go b/pkg/install/install.go index 1f5c1fecc..850911e9c 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -29,7 +29,7 @@ import ( "github.com/rancher/elemental-operator/pkg/util" agent "github.com/rancher/system-agent/pkg/config" "github.com/twpayne/go-vfs" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" From bdacd9ada026beaaacc9abd738f44ca56f0f55f9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 13:34:08 +0200 Subject: [PATCH 28/50] Explicitly cleanup local reset plan after execution --- controllers/machineinventory_controller.go | 4 +++- pkg/install/install.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index ab87793f5..3399d89fc 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -51,6 +51,8 @@ import ( // Timeout to validate machine inventory adoption const adoptionTimeout = 5 +const LocalResetPlanPath = "/oem/reset-plan.yaml" + // MachineInventoryReconciler reconciles a MachineInventory object. type MachineInventoryReconciler struct { client.Client @@ -304,7 +306,7 @@ func (r *MachineInventoryReconciler) newResetPlan(ctx context.Context) (string, Files: []applyinator.File{ { Content: base64.StdEncoding.EncodeToString(resetCloudConfigBytes), - Path: "/oem/reset-plan.yaml", + Path: LocalResetPlanPath, Permissions: "0600", }, }, diff --git a/pkg/install/install.go b/pkg/install/install.go index 850911e9c..c94c8a5ed 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -19,10 +19,12 @@ package install import ( "encoding/json" "fmt" + "os" "path/filepath" "github.com/mudler/yip/pkg/schema" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" + "github.com/rancher/elemental-operator/controllers" "github.com/rancher/elemental-operator/pkg/elementalcli" "github.com/rancher/elemental-operator/pkg/log" "github.com/rancher/elemental-operator/pkg/register" @@ -103,6 +105,10 @@ func (i *installer) ResetElemental(config elementalv1.Config) error { return fmt.Errorf("failed to reset elemental: %w", err) } + if err := i.cleanupResetPlan(); err != nil { + return fmt.Errorf("cleaning up reset plan: %w", err) + } + log.Info("Elemental reset completed, please reboot") return nil } @@ -299,3 +305,14 @@ func (i *installer) writeSystemAgentConfig(config elementalv1.Elemental) (string return f.Name(), err } + +func (i *installer) cleanupResetPlan() error { + _, err := i.fs.Stat(controllers.LocalResetPlanPath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("checking presence of file '%s': %w", controllers.LocalResetPlanPath, err) + } + if os.IsNotExist(err) { + log.Debugf("local reset plan '%s' does not exist, nothing to do", controllers.LocalResetPlanPath) + } + return i.fs.Remove(controllers.LocalResetPlanPath) +} From 7c98c53101826c23d154bd61ccecae39ec4d563e Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 13:34:58 +0200 Subject: [PATCH 29/50] Actually, cleanup the plan before execution, as reboot may occur --- pkg/install/install.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/install/install.go b/pkg/install/install.go index c94c8a5ed..fb14035d4 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -101,14 +101,14 @@ func (i *installer) ResetElemental(config elementalv1.Config) error { } config.Elemental.Reset.ConfigURLs = append(config.Elemental.Reset.ConfigURLs, additionalConfigs...) - if err := i.runner.Reset(config.Elemental.Reset); err != nil { - return fmt.Errorf("failed to reset elemental: %w", err) - } - if err := i.cleanupResetPlan(); err != nil { return fmt.Errorf("cleaning up reset plan: %w", err) } + if err := i.runner.Reset(config.Elemental.Reset); err != nil { + return fmt.Errorf("failed to reset elemental: %w", err) + } + log.Info("Elemental reset completed, please reboot") return nil } From 80255d79e3dc665d999551b784376ca2f708fb0a Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 13:38:34 +0200 Subject: [PATCH 30/50] Revert "Actually, cleanup the plan before execution, as reboot may occur" This reverts commit 7c98c53101826c23d154bd61ccecae39ec4d563e. --- pkg/install/install.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/install/install.go b/pkg/install/install.go index fb14035d4..c94c8a5ed 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -101,14 +101,14 @@ func (i *installer) ResetElemental(config elementalv1.Config) error { } config.Elemental.Reset.ConfigURLs = append(config.Elemental.Reset.ConfigURLs, additionalConfigs...) - if err := i.cleanupResetPlan(); err != nil { - return fmt.Errorf("cleaning up reset plan: %w", err) - } - if err := i.runner.Reset(config.Elemental.Reset); err != nil { return fmt.Errorf("failed to reset elemental: %w", err) } + if err := i.cleanupResetPlan(); err != nil { + return fmt.Errorf("cleaning up reset plan: %w", err) + } + log.Info("Elemental reset completed, please reboot") return nil } From fddc931dbc6335aaa82db04d9ebfd1a416156c44 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 15:33:39 +0200 Subject: [PATCH 31/50] Persist registration config on registration update --- cmd/register/main.go | 13 +++++++++---- pkg/install/install.go | 17 +++++++++++++++++ pkg/install/mocks/installer.go | 14 ++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index 402ea7045..a7994cb5d 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -103,6 +103,12 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err != nil { return fmt.Errorf("validating CA: %w", err) } + // Reset + // TODO: Add `MsgReset` to protocol so that it is possible to fetch the remote MachineRegistration + if reset { + log.Info("Resetting Elemental") + return installer.ResetElemental(cfg) + } // Register (and fetch the remote MachineRegistration) data, err := client.Register(cfg.Elemental.Registration, caCert, ®istrationState) if err != nil { @@ -123,10 +129,9 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa return fmt.Errorf("installing elemental: %w", err) } } - // Reset - if reset { - log.Info("Resetting Elemental") - return installer.ResetElemental(cfg) + // Persist config in default path + if err := installer.WriteConfig(cfg); err != nil { + return fmt.Errorf("persisting updated configuration: %w", err) } return nil diff --git a/pkg/install/install.go b/pkg/install/install.go index c94c8a5ed..6a0d9dc38 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -46,6 +46,7 @@ const ( ) type Installer interface { + WriteConfig(config elementalv1.Config) error ResetElemental(config elementalv1.Config) error InstallElemental(config elementalv1.Config, state register.State) error } @@ -113,6 +114,22 @@ func (i *installer) ResetElemental(config elementalv1.Config) error { return nil } +func (i *installer) WriteConfig(config elementalv1.Config) error { + // Since the full config may contain sensitive info (ex. system agent token), + // only persist what we actually need. + trimmedConf := elementalv1.Config{ + Elemental: elementalv1.Elemental{ + Registration: config.Elemental.Registration, + Reset: config.Elemental.Reset, + }, + } + configBytes, err := yaml.Marshal(trimmedConf) + if err != nil { + return fmt.Errorf("marshalling elemental config: %w", err) + } + return i.fs.WriteFile(registrationConf, configBytes, os.FileMode(600)) +} + func (i *installer) getCloudInitConfigs(config elementalv1.Config) ([]string, error) { configs := []string{} agentConfPath, err := i.writeSystemAgentConfig(config.Elemental) diff --git a/pkg/install/mocks/installer.go b/pkg/install/mocks/installer.go index a6c1ff107..9eb3126d1 100644 --- a/pkg/install/mocks/installer.go +++ b/pkg/install/mocks/installer.go @@ -62,3 +62,17 @@ func (mr *MockInstallerMockRecorder) ResetElemental(arg0 interface{}) *gomock.Ca mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetElemental", reflect.TypeOf((*MockInstaller)(nil).ResetElemental), arg0) } + +// WriteConfig mocks base method. +func (m *MockInstaller) WriteConfig(arg0 v1beta1.Config) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WriteConfig", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// WriteConfig indicates an expected call of WriteConfig. +func (mr *MockInstallerMockRecorder) WriteConfig(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteConfig", reflect.TypeOf((*MockInstaller)(nil).WriteConfig), arg0) +} From 42606edfcabdd7c4834bd19169f5ef3fc4319718 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 31 Jul 2023 16:16:37 +0200 Subject: [PATCH 32/50] Add debug logging --- cmd/register/main.go | 1 + pkg/elementalcli/elementalcli.go | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index a7994cb5d..282bb9600 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -130,6 +130,7 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa } } // Persist config in default path + log.Debug("Persisting configuration") if err := installer.WriteConfig(cfg); err != nil { return fmt.Errorf("persisting updated configuration: %w", err) } diff --git a/pkg/elementalcli/elementalcli.go b/pkg/elementalcli/elementalcli.go index 556ca9067..13a63ed9c 100644 --- a/pkg/elementalcli/elementalcli.go +++ b/pkg/elementalcli/elementalcli.go @@ -24,6 +24,7 @@ import ( "strings" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" + "github.com/rancher/elemental-operator/pkg/log" ) type Runner interface { @@ -52,11 +53,16 @@ func (r *runner) Install(conf elementalv1.Install) error { installerOpts = append(installerOpts, "install") cmd := exec.Command("elemental") - cmd.Env = append(os.Environ(), mapToInstallEnv(conf)...) + environmentVariables := mapToInstallEnv(conf) + cmd.Env = append(os.Environ(), environmentVariables...) cmd.Stdout = os.Stdout cmd.Args = installerOpts cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr + log.Debugf("running elemental %s", strings.Join(installerOpts, " ")) + for _, env := range environmentVariables { + log.Debug(env) + } return cmd.Run() } @@ -70,11 +76,16 @@ func (r *runner) Reset(conf elementalv1.Reset) error { installerOpts = append(installerOpts, "reset") cmd := exec.Command("elemental") - cmd.Env = append(os.Environ(), mapToResetEnv(conf)...) + environmentVariables := mapToResetEnv(conf) + cmd.Env = append(os.Environ(), environmentVariables...) cmd.Stdout = os.Stdout cmd.Args = installerOpts cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr + log.Debugf("running elemental %s", strings.Join(installerOpts, " ")) + for _, env := range environmentVariables { + log.Debug(env) + } return cmd.Run() } From 7d9d44951becedc4b7e159f4be064488ca72fd1a Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Tue, 1 Aug 2023 11:27:29 +0200 Subject: [PATCH 33/50] Fix status plan not being reset correctly. Add more verbosity to elementalcli calls --- api/v1beta1/types.go | 16 +++++++------- cmd/register/main.go | 1 + controllers/machineinventory_controller.go | 3 ++- pkg/elementalcli/elementalcli.go | 10 ++------- pkg/install/install.go | 25 ++++++++++++---------- pkg/server/api_registration.go | 1 + 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 7db3acb6e..3b9b334e6 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -51,24 +51,24 @@ type Install struct { type Reset struct { // +optional - Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty" mapstructure:"enabled"` // +optional // +kubebuilder:default:=true - ResetPersistent bool `json:"reset-persistent,omitempty" yaml:"reset-persistent,omitempty"` + ResetPersistent bool `json:"reset-persistent,omitempty" yaml:"reset-persistent,omitempty" mapstructure:"reset-persistent"` // +optional // +kubebuilder:default:=true - ResetOEM bool `json:"reset-oem,omitempty" yaml:"reset-oem,omitempty"` + ResetOEM bool `json:"reset-oem,omitempty" yaml:"reset-oem,omitempty" mapstructure:"reset-oem"` // +optional - ConfigURLs []string `json:"config-urls,omitempty" yaml:"config-urls,omitempty"` + ConfigURLs []string `json:"config-urls,omitempty" yaml:"config-urls,omitempty" mapstructure:"config-urls"` // +optional - SystemURI string `json:"system-uri,omitempty" yaml:"system-uri,omitempty"` + SystemURI string `json:"system-uri,omitempty" yaml:"system-uri,omitempty" mapstructure:"system-uri"` // +optional - Debug bool `json:"debug,omitempty" yaml:"debug,omitempty"` + Debug bool `json:"debug,omitempty" yaml:"debug,omitempty" mapstructure:"debug"` // +optional - PowerOff bool `json:"poweroff,omitempty" yaml:"poweroff,omitempty"` + PowerOff bool `json:"poweroff,omitempty" yaml:"poweroff,omitempty" mapstructure:"poweroff"` // +optional // +kubebuilder:default:=true - Reboot bool `json:"reboot,omitempty" yaml:"reboot,omitempty"` + Reboot bool `json:"reboot,omitempty" yaml:"reboot,omitempty" mapstructure:"reboot"` } type Registration struct { diff --git a/cmd/register/main.go b/cmd/register/main.go index 282bb9600..b573154e3 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -107,6 +107,7 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa // TODO: Add `MsgReset` to protocol so that it is possible to fetch the remote MachineRegistration if reset { log.Info("Resetting Elemental") + log.Debugf("Using config: %+v", cfg) return installer.ResetElemental(cfg) } // Register (and fetch the remote MachineRegistration) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 3399d89fc..4bfa66fd8 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -127,7 +127,6 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * if err := r.Update(ctx, mInventory); err != nil { return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) } - return ctrl.Result{RequeueAfter: time.Second}, nil } } else { // The object is up for deletion @@ -257,6 +256,7 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte return fmt.Errorf("patching plan secret: %w", err) } + // Clear the plan status mInventory.Status.Plan = &elementalv1.PlanStatus{ Checksum: checksum, PlanSecretRef: &corev1.ObjectReference{ @@ -387,6 +387,7 @@ func (r *MachineInventoryReconciler) createPlanSecret(ctx context.Context, mInve Namespace: planSecret.Namespace, Name: planSecret.Name, }, + State: elementalv1.PlanState(""), } logger.Info("Plan secret created") diff --git a/pkg/elementalcli/elementalcli.go b/pkg/elementalcli/elementalcli.go index 13a63ed9c..806bd1b0c 100644 --- a/pkg/elementalcli/elementalcli.go +++ b/pkg/elementalcli/elementalcli.go @@ -59,10 +59,7 @@ func (r *runner) Install(conf elementalv1.Install) error { cmd.Args = installerOpts cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr - log.Debugf("running elemental %s", strings.Join(installerOpts, " ")) - for _, env := range environmentVariables { - log.Debug(env) - } + log.Debugf("running: %s\n with ENV:\n%s", strings.Join(installerOpts, " "), strings.Join(environmentVariables, "\n")) return cmd.Run() } @@ -82,10 +79,7 @@ func (r *runner) Reset(conf elementalv1.Reset) error { cmd.Args = installerOpts cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr - log.Debugf("running elemental %s", strings.Join(installerOpts, " ")) - for _, env := range environmentVariables { - log.Debug(env) - } + log.Debugf("running: %s\n with ENV:\n%s", strings.Join(installerOpts, " "), strings.Join(environmentVariables, "\n")) return cmd.Run() } diff --git a/pkg/install/install.go b/pkg/install/install.go index 6a0d9dc38..ad1bfaff8 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -115,19 +115,21 @@ func (i *installer) ResetElemental(config elementalv1.Config) error { } func (i *installer) WriteConfig(config elementalv1.Config) error { - // Since the full config may contain sensitive info (ex. system agent token), - // only persist what we actually need. - trimmedConf := elementalv1.Config{ - Elemental: elementalv1.Elemental{ - Registration: config.Elemental.Registration, - Reset: config.Elemental.Reset, - }, - } - configBytes, err := yaml.Marshal(trimmedConf) + file, err := i.fs.Create(registrationConf) if err != nil { - return fmt.Errorf("marshalling elemental config: %w", err) + return fmt.Errorf("creating registration config file: %w", err) + } + enc := yaml.NewEncoder(file) + if err := enc.Encode(config); err != nil { + return fmt.Errorf("writing registration config to file '%s': %w", registrationConf, err) } - return i.fs.WriteFile(registrationConf, configBytes, os.FileMode(600)) + if err := enc.Close(); err != nil { + return fmt.Errorf("closing encoder: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("closing file '%s': %w", registrationConf, err) + } + return nil } func (i *installer) getCloudInitConfigs(config elementalv1.Config) ([]string, error) { @@ -175,6 +177,7 @@ func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, Stages: map[string][]schema.Stage{ "initramfs": { schema.Stage{ + If: fmt.Sprintf("[ ! -f %s ]", registrationConf), Directories: []schema.Directory{ { Path: filepath.Dir(registrationConf), diff --git a/pkg/server/api_registration.go b/pkg/server/api_registration.go index aae80d18f..d4049851a 100644 --- a/pkg/server/api_registration.go +++ b/pkg/server/api_registration.go @@ -171,6 +171,7 @@ func (i *InventoryServer) writeMachineInventoryCloudConfig(conn *websocket.Conn, SecretNamespace: inventory.Namespace, }, Install: registration.Spec.Config.Elemental.Install, + Reset: registration.Spec.Config.Elemental.Reset, } cloudConf := registration.Spec.Config.CloudConfig From 8c5dc9c3ad46101381a73ebeecc04cc93fdf2c26 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 09:25:37 +0200 Subject: [PATCH 34/50] Register using new state when resetting --- cmd/register/main.go | 31 ++++++++++++++----------- cmd/register/main_test.go | 8 +++---- pkg/install/install.go | 41 +++++++++------------------------- pkg/install/mocks/installer.go | 22 ++++-------------- 4 files changed, 37 insertions(+), 65 deletions(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index b573154e3..f5ab47e98 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -89,9 +89,9 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err := stateHandler.Init(statePath); err != nil { return fmt.Errorf("initializing state handler on path '%s': %w", statePath, err) } - registrationState, err := stateHandler.Load() + registrationState, err := getRegistrationState(stateHandler, reset) if err != nil { - return fmt.Errorf("loading registration state: %w", err) + return fmt.Errorf("getting registration state: %w", err) } // Determine if registration should execute or skip a cycle if !installation && !reset && !registrationState.HasLastUpdateElapsed(registrationUpdateSuppressTimer) { @@ -103,13 +103,6 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err != nil { return fmt.Errorf("validating CA: %w", err) } - // Reset - // TODO: Add `MsgReset` to protocol so that it is possible to fetch the remote MachineRegistration - if reset { - log.Info("Resetting Elemental") - log.Debugf("Using config: %+v", cfg) - return installer.ResetElemental(cfg) - } // Register (and fetch the remote MachineRegistration) data, err := client.Register(cfg.Elemental.Registration, caCert, ®istrationState) if err != nil { @@ -130,10 +123,10 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa return fmt.Errorf("installing elemental: %w", err) } } - // Persist config in default path - log.Debug("Persisting configuration") - if err := installer.WriteConfig(cfg); err != nil { - return fmt.Errorf("persisting updated configuration: %w", err) + // Reset + if reset { + log.Info("Resetting Elemental") + return installer.ResetElemental(cfg, registrationState) } return nil @@ -162,6 +155,18 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa return cmd } +func getRegistrationState(stateHandler register.StateHandler, reset bool) (register.State, error) { + // If we are resetting, we create an empty state to perform an initial registration. + if reset { + return register.State{}, nil + } + registrationState, err := stateHandler.Load() + if err != nil { + return register.State{}, fmt.Errorf("loading registration state: %w", err) + } + return registrationState, nil +} + func initConfig(fs vfs.FS) error { log.Infof("Register version %s, commit %s, commit date %s", version.Version, version.Commit, version.CommitDate) if installation && reset { diff --git a/cmd/register/main_test.go b/cmd/register/main_test.go index fc4855cec..47c89fb15 100644 --- a/cmd/register/main_test.go +++ b/cmd/register/main_test.go @@ -275,14 +275,14 @@ var _ = Describe("elemental-register --reset", Label("registration", "cli", "res BeforeEach(func() { marshalIntoFile(fs, baseConfigFixture, defaultConfigPath) stateHandler.EXPECT().Init(defaultStatePath).Return(nil) - stateHandler.EXPECT().Load().Return(stateFixture, nil) - stateHandler.EXPECT().Save(stateFixture).Return(nil) + stateHandler.EXPECT().Load().Times(0) // When resetting expect new state to be initialized + stateHandler.EXPECT().Save(register.State{}).Return(nil) }) It("should trigger reset when --reset argument", func() { cmd.SetArgs([]string{"--reset"}) - installer.EXPECT().ResetElemental(alternateConfigFixture).Return(nil) + installer.EXPECT().ResetElemental(alternateConfigFixture, register.State{}).Return(nil) client.EXPECT(). - Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert), &stateFixture). + Register(baseConfigFixture.Elemental.Registration, []byte(baseConfigFixture.Elemental.Registration.CACert), ®ister.State{}). Return(marshalToBytes(alternateConfigFixture), nil) Expect(cmd.Execute()).ToNot(HaveOccurred()) }) diff --git a/pkg/install/install.go b/pkg/install/install.go index ad1bfaff8..8b89588e2 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -46,8 +46,7 @@ const ( ) type Installer interface { - WriteConfig(config elementalv1.Config) error - ResetElemental(config elementalv1.Config) error + ResetElemental(config elementalv1.Config, state register.State) error InstallElemental(config elementalv1.Config, state register.State) error } @@ -70,16 +69,10 @@ func (i *installer) InstallElemental(config elementalv1.Config, state register.S config.Elemental.Install.ConfigURLs = []string{} } - additionalConfigs, err := i.getCloudInitConfigs(config) + additionalConfigs, err := i.getCloudInitConfigs(config, state) if err != nil { return fmt.Errorf("generating additional cloud configs: %w", err) } - registrationStatePath, err := i.writeRegistrationState(state) - - if err != nil { - return fmt.Errorf("writing registration state plan: %w", err) - } - additionalConfigs = append(additionalConfigs, registrationStatePath) config.Elemental.Install.ConfigURLs = append(config.Elemental.Install.ConfigURLs, additionalConfigs...) @@ -91,12 +84,12 @@ func (i *installer) InstallElemental(config elementalv1.Config, state register.S return nil } -func (i *installer) ResetElemental(config elementalv1.Config) error { +func (i *installer) ResetElemental(config elementalv1.Config, state register.State) error { if config.Elemental.Reset.ConfigURLs == nil { config.Elemental.Reset.ConfigURLs = []string{} } - additionalConfigs, err := i.getCloudInitConfigs(config) + additionalConfigs, err := i.getCloudInitConfigs(config, state) if err != nil { return fmt.Errorf("generating additional cloud configs: %w", err) } @@ -114,25 +107,7 @@ func (i *installer) ResetElemental(config elementalv1.Config) error { return nil } -func (i *installer) WriteConfig(config elementalv1.Config) error { - file, err := i.fs.Create(registrationConf) - if err != nil { - return fmt.Errorf("creating registration config file: %w", err) - } - enc := yaml.NewEncoder(file) - if err := enc.Encode(config); err != nil { - return fmt.Errorf("writing registration config to file '%s': %w", registrationConf, err) - } - if err := enc.Close(); err != nil { - return fmt.Errorf("closing encoder: %w", err) - } - if err := file.Close(); err != nil { - return fmt.Errorf("closing file '%s': %w", registrationConf, err) - } - return nil -} - -func (i *installer) getCloudInitConfigs(config elementalv1.Config) ([]string, error) { +func (i *installer) getCloudInitConfigs(config elementalv1.Config, state register.State) ([]string, error) { configs := []string{} agentConfPath, err := i.writeSystemAgentConfig(config.Elemental) if err != nil { @@ -154,6 +129,12 @@ func (i *installer) getCloudInitConfigs(config elementalv1.Config) ([]string, er } configs = append(configs, registrationConfPath) + registrationStatePath, err := i.writeRegistrationState(state) + if err != nil { + return nil, fmt.Errorf("writing registration state plan: %w", err) + } + configs = append(configs, registrationStatePath) + return configs, nil } diff --git a/pkg/install/mocks/installer.go b/pkg/install/mocks/installer.go index 9eb3126d1..e6d89f787 100644 --- a/pkg/install/mocks/installer.go +++ b/pkg/install/mocks/installer.go @@ -50,29 +50,15 @@ func (mr *MockInstallerMockRecorder) InstallElemental(arg0, arg1 interface{}) *g } // ResetElemental mocks base method. -func (m *MockInstaller) ResetElemental(arg0 v1beta1.Config) error { +func (m *MockInstaller) ResetElemental(arg0 v1beta1.Config, arg1 register.State) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResetElemental", arg0) + ret := m.ctrl.Call(m, "ResetElemental", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // ResetElemental indicates an expected call of ResetElemental. -func (mr *MockInstallerMockRecorder) ResetElemental(arg0 interface{}) *gomock.Call { +func (mr *MockInstallerMockRecorder) ResetElemental(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetElemental", reflect.TypeOf((*MockInstaller)(nil).ResetElemental), arg0) -} - -// WriteConfig mocks base method. -func (m *MockInstaller) WriteConfig(arg0 v1beta1.Config) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WriteConfig", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// WriteConfig indicates an expected call of WriteConfig. -func (mr *MockInstallerMockRecorder) WriteConfig(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteConfig", reflect.TypeOf((*MockInstaller)(nil).WriteConfig), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetElemental", reflect.TypeOf((*MockInstaller)(nil).ResetElemental), arg0, arg1) } From 4b2e7a9096c2071eac05c25792b49f0e7974623d Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 10:00:56 +0200 Subject: [PATCH 35/50] Do not update Minventory directly in inner reconcile loop --- controllers/machineinventory_controller.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 4bfa66fd8..129335c9d 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -124,9 +124,10 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * // The object is not being deleted, so register the finalizer if !controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { controllerutil.AddFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) - if err := r.Update(ctx, mInventory); err != nil { - return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) - } + return ctrl.Result{RequeueAfter: time.Second}, nil + // if err := r.Update(ctx, mInventory); err != nil { + // return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) + // } } } else { // The object is up for deletion From 407c28fce3d9df8fb06d0a84d90dd56a2e9b88ab Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 10:20:17 +0200 Subject: [PATCH 36/50] Fix error message --- controllers/machineinventory_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 129335c9d..3d62613f1 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -99,7 +99,7 @@ func (r *MachineInventoryReconciler) Reconcile(ctx context.Context, req reconcil machineInventoryStatusCopy := mInventory.Status.DeepCopy() // Patch call will erase the status if err := r.Patch(ctx, mInventory, patchBase); err != nil { - errs = append(errs, fmt.Errorf("failed to patch status for machine inventory object: %w", err)) + errs = append(errs, fmt.Errorf("failed to patch machine inventory object: %w", err)) } mInventory.Status = *machineInventoryStatusCopy From df94576aeb58012892de389bf18db974cb4334d3 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 10:25:30 +0200 Subject: [PATCH 37/50] Add finalizer in test MInventory --- controllers/machineinventory_controller.go | 3 --- controllers/machineinventory_controller_test.go | 5 +++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 3d62613f1..983fb083a 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -125,9 +125,6 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * if !controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { controllerutil.AddFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) return ctrl.Result{RequeueAfter: time.Second}, nil - // if err := r.Update(ctx, mInventory); err != nil { - // return ctrl.Result{}, fmt.Errorf("updating machine inventory finalizer: %w", err) - // } } } else { // The object is up for deletion diff --git a/controllers/machineinventory_controller_test.go b/controllers/machineinventory_controller_test.go index 539c1dc13..d91818103 100644 --- a/controllers/machineinventory_controller_test.go +++ b/controllers/machineinventory_controller_test.go @@ -47,8 +47,9 @@ var _ = Describe("reconcile machine inventory", func() { mInventory = &elementalv1.MachineInventory{ ObjectMeta: metav1.ObjectMeta{ - Name: "machine-inventory-suite", - Namespace: "default", + Finalizers: []string{elementalv1.MachineInventoryFinalizer}, + Name: "machine-inventory-suite", + Namespace: "default", }, } From a1c4a51a42e0cceb1bbb13c4a80fb1043348d412 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 12:40:43 +0200 Subject: [PATCH 38/50] Add elemental installer test coverage --- pkg/elementalcli/mocks/elementalcli.go | 63 +++++++ pkg/install/_testdata/cloud-init-config.yaml | 4 + .../_testdata/registration-config-config.yaml | 25 +++ .../_testdata/registration-state-config.yaml | 20 ++ .../_testdata/system-agent-config.yaml | 18 ++ pkg/install/install.go | 53 ++++-- pkg/install/install_test.go | 173 ++++++++++++++++++ .../mocks/{installer.go => install.go} | 0 scripts/generate_mocks.sh | 3 +- 9 files changed, 341 insertions(+), 18 deletions(-) create mode 100644 pkg/elementalcli/mocks/elementalcli.go create mode 100644 pkg/install/_testdata/cloud-init-config.yaml create mode 100644 pkg/install/_testdata/registration-config-config.yaml create mode 100644 pkg/install/_testdata/registration-state-config.yaml create mode 100644 pkg/install/_testdata/system-agent-config.yaml create mode 100644 pkg/install/install_test.go rename pkg/install/mocks/{installer.go => install.go} (100%) diff --git a/pkg/elementalcli/mocks/elementalcli.go b/pkg/elementalcli/mocks/elementalcli.go new file mode 100644 index 000000000..cb81b6058 --- /dev/null +++ b/pkg/elementalcli/mocks/elementalcli.go @@ -0,0 +1,63 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/rancher/elemental-operator/pkg/elementalcli (interfaces: Runner) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + v1beta1 "github.com/rancher/elemental-operator/api/v1beta1" + gomock "go.uber.org/mock/gomock" +) + +// MockRunner is a mock of Runner interface. +type MockRunner struct { + ctrl *gomock.Controller + recorder *MockRunnerMockRecorder +} + +// MockRunnerMockRecorder is the mock recorder for MockRunner. +type MockRunnerMockRecorder struct { + mock *MockRunner +} + +// NewMockRunner creates a new mock instance. +func NewMockRunner(ctrl *gomock.Controller) *MockRunner { + mock := &MockRunner{ctrl: ctrl} + mock.recorder = &MockRunnerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRunner) EXPECT() *MockRunnerMockRecorder { + return m.recorder +} + +// Install mocks base method. +func (m *MockRunner) Install(arg0 v1beta1.Install) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Install", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Install indicates an expected call of Install. +func (mr *MockRunnerMockRecorder) Install(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Install", reflect.TypeOf((*MockRunner)(nil).Install), arg0) +} + +// Reset mocks base method. +func (m *MockRunner) Reset(arg0 v1beta1.Reset) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Reset", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Reset indicates an expected call of Reset. +func (mr *MockRunnerMockRecorder) Reset(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reset", reflect.TypeOf((*MockRunner)(nil).Reset), arg0) +} diff --git a/pkg/install/_testdata/cloud-init-config.yaml b/pkg/install/_testdata/cloud-init-config.yaml new file mode 100644 index 000000000..fb49c03df --- /dev/null +++ b/pkg/install/_testdata/cloud-init-config.yaml @@ -0,0 +1,4 @@ +#cloud-config +users: +- name: root + passwd: root diff --git a/pkg/install/_testdata/registration-config-config.yaml b/pkg/install/_testdata/registration-config-config.yaml new file mode 100644 index 000000000..557cbc5a4 --- /dev/null +++ b/pkg/install/_testdata/registration-config-config.yaml @@ -0,0 +1,25 @@ +name: Include registration config into installed system +stages: + initramfs: + - files: + - path: /oem/registration/config.yaml + permissions: 384 + owner: 0 + group: 0 + content: | + elemental: + registration: + url: https://127.0.0.1.sslip.io/test/registration/endpoint + ca-cert: a test ca + emulate-tpm: true + emulated-tpm-seed: 9876543210 + no-smbios: true + auth: a test auth + encoding: "" + ownerstring: "" + directories: + - path: /oem/registration + permissions: 448 + owner: 0 + group: 0 + if: '[ ! -f /oem/registration/config.yaml ]' diff --git a/pkg/install/_testdata/registration-state-config.yaml b/pkg/install/_testdata/registration-state-config.yaml new file mode 100644 index 000000000..0feb15876 --- /dev/null +++ b/pkg/install/_testdata/registration-state-config.yaml @@ -0,0 +1,20 @@ +name: Include registration state into installed system +stages: + initramfs: + - files: + - path: /oem/registration/state.yaml + permissions: 384 + owner: 0 + group: 0 + content: | + initialRegistration: 2023-08-02T12:35:10.000000003Z + emulatedTPM: true + emulatedTPMSeed: 987654321 + encoding: "" + ownerstring: "" + directories: + - path: /oem/registration + permissions: 448 + owner: 0 + group: 0 + if: '[ ! -f /oem/registration/state.yaml ]' diff --git a/pkg/install/_testdata/system-agent-config.yaml b/pkg/install/_testdata/system-agent-config.yaml new file mode 100644 index 000000000..e7acea5fe --- /dev/null +++ b/pkg/install/_testdata/system-agent-config.yaml @@ -0,0 +1,18 @@ +name: Elemental System Agent Configuration +stages: + initramfs: + - files: + - path: /var/lib/elemental/agent/elemental_connection.json + permissions: 384 + owner: 0 + group: 0 + content: '{"kubeConfig":"apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: YSB0ZXN0IGNh\n server: https://127.0.0.1.sslip.io/test/control/plane/endpoint\n name: cluster\ncontexts:\n- context:\n cluster: cluster\n user: user\n name: context\ncurrent-context: context\nkind: Config\npreferences: {}\nusers:\n- name: user\n user:\n token: a test token\n","namespace":"a test namespace","secretName":"a test secret name"}' + encoding: "" + ownerstring: "" + - path: /etc/rancher/elemental/agent/config.yaml + permissions: 384 + owner: 0 + group: 0 + content: '{"workDirectory":"/var/lib/elemental/agent/work","localEnabled":true,"localPlanDirectory":"/var/lib/elemental/agent/plans","appliedPlanDirectory":"/var/lib/elemental/agent/applied","remoteEnabled":true,"connectionInfoFile":"/var/lib/elemental/agent/elemental_connection.json"}' + encoding: "" + ownerstring: "" diff --git a/pkg/install/install.go b/pkg/install/install.go index 8b89588e2..9fed08b56 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -45,6 +45,15 @@ const ( registrationState = "/oem/registration/state.yaml" ) +// Temporary cloud-init configuration files. +// These paths will be passed to the `elemental` cli as additional `config-urls`. +const ( + tempRegistrationConf = "/tmp/elemental-registration-conf.yaml" + tempRegistrationState = "/tmp/elemental-registration-state.yaml" + tempCloudInit = "/tmp/elemental-cloud-init.yaml" + tempSystemAgent = "/tmp/elemental-system-agent.yaml" +) + type Installer interface { ResetElemental(config elementalv1.Config, state register.State) error InstallElemental(config elementalv1.Config, state register.State) error @@ -107,6 +116,9 @@ func (i *installer) ResetElemental(config elementalv1.Config, state register.Sta return nil } +// getCloudInitConfigs creates cloud-init configuration files that can be passed as additional `config-urls` +// to the `elemental` cli. We exploit this mechanism to persist information during `elemental install` +// or `elemental reset` calls into the newly installed or resetted system. func (i *installer) getCloudInitConfigs(config elementalv1.Config, state register.State) ([]string, error) { configs := []string{} agentConfPath, err := i.writeSystemAgentConfig(config.Elemental) @@ -139,7 +151,7 @@ func (i *installer) getCloudInitConfigs(config elementalv1.Config, state registe } func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, error) { - f, err := i.fs.Create("/tmp/elemental-registration-conf.yaml") + f, err := i.fs.Create(tempRegistrationConf) if err != nil { return "", fmt.Errorf("creating temporary registration conf plan file: %w", err) } @@ -153,7 +165,7 @@ func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, return "", fmt.Errorf("marshalling registration config: %w", err) } - err = yaml.NewEncoder(f).Encode(schema.YipConfig{ + if err := yaml.NewEncoder(f).Encode(schema.YipConfig{ Name: "Include registration config into installed system", Stages: map[string][]schema.Stage{ "initramfs": { @@ -174,13 +186,14 @@ func (i *installer) writeRegistrationYAML(reg elementalv1.Registration) (string, }, }, }, - }) - - return f.Name(), err + }); err != nil { + return "", fmt.Errorf("writing encoded registration config config: %w", err) + } + return f.Name(), nil } func (i *installer) writeRegistrationState(state register.State) (string, error) { - f, err := i.fs.Create("/tmp/elemental-registration-state.yaml") + f, err := i.fs.Create(tempRegistrationState) if err != nil { return "", fmt.Errorf("creating temporary registration state plan file: %w", err) } @@ -190,7 +203,7 @@ func (i *installer) writeRegistrationState(state register.State) (string, error) return "", fmt.Errorf("marshalling registration state: %w", err) } - err = yaml.NewEncoder(f).Encode(schema.YipConfig{ + if err := yaml.NewEncoder(f).Encode(schema.YipConfig{ Name: "Include registration state into installed system", Stages: map[string][]schema.Stage{ "initramfs": { @@ -211,12 +224,14 @@ func (i *installer) writeRegistrationState(state register.State) (string, error) }, }, }, - }) - return f.Name(), err + }); err != nil { + return "", fmt.Errorf("writing encoded registration state config: %w", err) + } + return f.Name(), nil } func (i *installer) writeCloudInit(cloudConfig map[string]runtime.RawExtension) (string, error) { - f, err := i.fs.Create("/tmp/elemental-cloud-init.yaml") + f, err := i.fs.Create(tempCloudInit) if err != nil { return "", fmt.Errorf("creating temporary cloud init file: %w", err) } @@ -228,8 +243,10 @@ func (i *installer) writeCloudInit(cloudConfig map[string]runtime.RawExtension) } log.Debugf("Decoded CloudConfig:\n%s\n", string(bytes)) - _, err = f.Write(bytes) - return f.Name(), err + if _, err = f.Write(bytes); err != nil { + return "", fmt.Errorf("writing cloud config: %w", err) + } + return f.Name(), nil } func (i *installer) writeSystemAgentConfig(config elementalv1.Elemental) (string, error) { @@ -291,20 +308,21 @@ func (i *installer) writeSystemAgentConfig(config elementalv1.Elemental) (string }, }) - f, err := i.fs.Create("/tmp/elemental-system-agent.yaml") + f, err := i.fs.Create(tempSystemAgent) if err != nil { return "", fmt.Errorf("creating temporary elemental-system-agent file: %w", err) } defer f.Close() - err = yaml.NewEncoder(f).Encode(schema.YipConfig{ + if err := yaml.NewEncoder(f).Encode(schema.YipConfig{ Name: "Elemental System Agent Configuration", Stages: map[string][]schema.Stage{ "initramfs": stages, }, - }) - - return f.Name(), err + }); err != nil { + return "", fmt.Errorf("writing encoded system agent config: %w", err) + } + return f.Name(), nil } func (i *installer) cleanupResetPlan() error { @@ -314,6 +332,7 @@ func (i *installer) cleanupResetPlan() error { } if os.IsNotExist(err) { log.Debugf("local reset plan '%s' does not exist, nothing to do", controllers.LocalResetPlanPath) + return nil } return i.fs.Remove(controllers.LocalResetPlanPath) } diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go new file mode 100644 index 000000000..b2a102c33 --- /dev/null +++ b/pkg/install/install_test.go @@ -0,0 +1,173 @@ +/* +Copyright © 2022 - 2023 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package install + +import ( + "fmt" + "io/ioutil" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/twpayne/go-vfs" + "github.com/twpayne/go-vfs/vfst" + "go.uber.org/mock/gomock" + "k8s.io/apimachinery/pkg/runtime" + + elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" + climocks "github.com/rancher/elemental-operator/pkg/elementalcli/mocks" + "github.com/rancher/elemental-operator/pkg/register" +) + +var ( + configFixture = elementalv1.Config{ + Elemental: elementalv1.Elemental{ + Registration: elementalv1.Registration{ + URL: "https://127.0.0.1.sslip.io/test/registration/endpoint", + CACert: "a test ca", + EmulateTPM: true, + EmulatedTPMSeed: 9876543210, + NoSMBIOS: true, + Auth: "a test auth", + }, + Install: elementalv1.Install{ + Firmware: "a test firmware", + Device: "a test device", + NoFormat: true, + ConfigURLs: []string{"foo", "bar"}, + ISO: "a test iso", + SystemURI: "a system uri", + Debug: true, + TTY: "a test tty", + PowerOff: true, + Reboot: true, + EjectCD: true, + DisableBootEntry: true, + ConfigDir: "a test config dir", + }, + Reset: elementalv1.Reset{ + Enabled: true, + ResetPersistent: false, + ResetOEM: false, + ConfigURLs: []string{"foo", "bar"}, + SystemURI: "a system uri", + PowerOff: true, + Reboot: true, + }, + SystemAgent: elementalv1.SystemAgent{ + URL: "https://127.0.0.1.sslip.io/test/control/plane/endpoint", + Token: "a test token", + SecretName: "a test secret name", + SecretNamespace: "a test namespace", + }, + }, + CloudConfig: map[string]runtime.RawExtension{ + "users": { + Raw: []byte(`[{"name":"root","passwd":"root"}]`), + }, + }, + } + stateFixture = register.State{ + InitialRegistration: time.Date(2023, time.August, 2, 12, 35, 10, 3, time.UTC), + LastUpdate: time.Time{}, + EmulatedTPM: true, + EmulatedTPMSeed: 987654321, + } +) + +func TestInstall(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Install Suite") +} + +var _ = Describe("installer install elemental", Label("installer", "install"), func() { + var fs *vfst.TestFS + var err error + var fsCleanup func() + var cliRunner *climocks.MockRunner + var install Installer + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{"/tmp/init": ""}) + Expect(err).ToNot(HaveOccurred()) + mockCtrl := gomock.NewController(GinkgoT()) + cliRunner = climocks.NewMockRunner(mockCtrl) + install = &installer{ + fs: fs, + runner: cliRunner, + } + DeferCleanup(fsCleanup) + }) + It("should call elemental install", func() { + wantConfig := configFixture.DeepCopy() + wantConfig.Elemental.Install.ConfigURLs = append(wantConfig.Elemental.Install.ConfigURLs, additionalConfigs(fs)...) + cliRunner.EXPECT().Install(wantConfig.Elemental.Install).Return(nil) + Expect(install.InstallElemental(configFixture, stateFixture)).ToNot(HaveOccurred()) + checkConfigs(fs) + }) +}) + +var _ = Describe("installer reset elemental", Label("installer", "reset"), func() { + var fs *vfst.TestFS + var err error + var fsCleanup func() + var cliRunner *climocks.MockRunner + var install Installer + BeforeEach(func() { + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{"/tmp/init": ""}) + Expect(err).ToNot(HaveOccurred()) + mockCtrl := gomock.NewController(GinkgoT()) + cliRunner = climocks.NewMockRunner(mockCtrl) + install = &installer{ + fs: fs, + runner: cliRunner, + } + DeferCleanup(fsCleanup) + }) + It("should call elemental install", func() { + wantConfig := configFixture.DeepCopy() + wantConfig.Elemental.Reset.ConfigURLs = append(wantConfig.Elemental.Reset.ConfigURLs, additionalConfigs(fs)...) + cliRunner.EXPECT().Reset(wantConfig.Elemental.Reset).Return(nil) + Expect(install.ResetElemental(configFixture, stateFixture)).ToNot(HaveOccurred()) + checkConfigs(fs) + }) +}) + +func additionalConfigs(fs *vfst.TestFS) []string { + // Prefix the go-vfs temp dir because that's what file.Name() returns + return []string{ + fmt.Sprintf("%s%s", fs.TempDir(), tempSystemAgent), + fmt.Sprintf("%s%s", fs.TempDir(), tempCloudInit), + fmt.Sprintf("%s%s", fs.TempDir(), tempRegistrationConf), + fmt.Sprintf("%s%s", fs.TempDir(), tempRegistrationState), + } +} + +func checkConfigs(fs vfs.FS) { + compareFiles(fs, tempRegistrationConf, "_testdata/registration-config-config.yaml") + compareFiles(fs, tempRegistrationState, "_testdata/registration-state-config.yaml") + compareFiles(fs, tempSystemAgent, "_testdata/system-agent-config.yaml") + compareFiles(fs, tempCloudInit, "_testdata/cloud-init-config.yaml") +} + +func compareFiles(fs vfs.FS, got string, want string) { + gotFile, err := fs.ReadFile(got) + Expect(err).ToNot(HaveOccurred()) + wantFile, err := ioutil.ReadFile(want) + Expect(err).ToNot(HaveOccurred()) + Expect(string(gotFile)).To(Equal(string(wantFile))) +} diff --git a/pkg/install/mocks/installer.go b/pkg/install/mocks/install.go similarity index 100% rename from pkg/install/mocks/installer.go rename to pkg/install/mocks/install.go diff --git a/scripts/generate_mocks.sh b/scripts/generate_mocks.sh index 4e7db6c50..42ee4e84e 100755 --- a/scripts/generate_mocks.sh +++ b/scripts/generate_mocks.sh @@ -7,4 +7,5 @@ go install go.uber.org/mock/mockgen@latest mockgen -destination=pkg/register/mocks/client.go -package=mocks github.com/rancher/elemental-operator/pkg/register Client mockgen -destination=pkg/register/mocks/state.go -package=mocks github.com/rancher/elemental-operator/pkg/register StateHandler -mockgen -destination=pkg/install/mocks/installer.go -package=mocks github.com/rancher/elemental-operator/pkg/install Installer +mockgen -destination=pkg/install/mocks/install.go -package=mocks github.com/rancher/elemental-operator/pkg/install Installer +mockgen -destination=pkg/elementalcli/mocks/elementalcli.go -package=mocks github.com/rancher/elemental-operator/pkg/elementalcli Runner From a19bebb7a230039d49296385fd55c298106c456c Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 13:14:48 +0200 Subject: [PATCH 39/50] Add remove reset plan test --- pkg/install/install_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/install/install_test.go b/pkg/install/install_test.go index b2a102c33..7ea2c904f 100644 --- a/pkg/install/install_test.go +++ b/pkg/install/install_test.go @@ -19,6 +19,7 @@ package install import ( "fmt" "io/ioutil" + "os" "testing" "time" @@ -30,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" + "github.com/rancher/elemental-operator/controllers" climocks "github.com/rancher/elemental-operator/pkg/elementalcli/mocks" "github.com/rancher/elemental-operator/pkg/register" ) @@ -128,7 +130,7 @@ var _ = Describe("installer reset elemental", Label("installer", "reset"), func( var cliRunner *climocks.MockRunner var install Installer BeforeEach(func() { - fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{"/tmp/init": ""}) + fs, fsCleanup, err = vfst.NewTestFS(map[string]interface{}{"/tmp/init": "", "/oem/init": ""}) Expect(err).ToNot(HaveOccurred()) mockCtrl := gomock.NewController(GinkgoT()) cliRunner = climocks.NewMockRunner(mockCtrl) @@ -138,13 +140,20 @@ var _ = Describe("installer reset elemental", Label("installer", "reset"), func( } DeferCleanup(fsCleanup) }) - It("should call elemental install", func() { + It("should call elemental reset", func() { wantConfig := configFixture.DeepCopy() wantConfig.Elemental.Reset.ConfigURLs = append(wantConfig.Elemental.Reset.ConfigURLs, additionalConfigs(fs)...) cliRunner.EXPECT().Reset(wantConfig.Elemental.Reset).Return(nil) Expect(install.ResetElemental(configFixture, stateFixture)).ToNot(HaveOccurred()) checkConfigs(fs) }) + It("should remove reset plan", func() { + Expect(fs.WriteFile(controllers.LocalResetPlanPath, []byte("{}\n"), os.FileMode(600))).ToNot(HaveOccurred()) + cliRunner.EXPECT().Reset(gomock.Any()).Return(nil) + Expect(install.ResetElemental(configFixture, stateFixture)).ToNot(HaveOccurred()) + _, err := fs.Stat(controllers.LocalResetPlanPath) + Expect(err).To(MatchError(os.ErrNotExist)) + }) }) func additionalConfigs(fs *vfst.TestFS) []string { From 216a485391f941f91262261f6c70d07e0c6cc6c6 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 15:46:41 +0200 Subject: [PATCH 40/50] Add finalizer handling tests --- controllers/machineinventory_controller.go | 11 +- .../machineinventory_controller_test.go | 158 ++++++++++++++++++ 2 files changed, 161 insertions(+), 8 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 983fb083a..172cd6ef3 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -104,7 +104,8 @@ func (r *MachineInventoryReconciler) Reconcile(ctx context.Context, req reconcil mInventory.Status = *machineInventoryStatusCopy - if err := r.Status().Patch(ctx, mInventory, patchBase); err != nil { + // If the object was waiting for deletion and we just removed the finalizer, we will get a not found error + if err := r.Status().Patch(ctx, mInventory, patchBase); err != nil && !apierrors.IsNotFound(err) { errs = append(errs, fmt.Errorf("failed to patch status for machine inventory object: %w", err)) } @@ -138,6 +139,7 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * }) return ctrl.Result{}, fmt.Errorf("reconciling reset plan secret: %w", err) } + return ctrl.Result{}, nil } return ctrl.Result{}, nil } @@ -263,13 +265,6 @@ func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Conte }, } - meta.SetStatusCondition(&mInventory.Status.Conditions, metav1.Condition{ - Type: elementalv1.ReadyCondition, - Reason: elementalv1.WaitingForPlanReason, - Status: metav1.ConditionFalse, - Message: "waiting for reset plan to be applied", - }) - return nil } diff --git a/controllers/machineinventory_controller_test.go b/controllers/machineinventory_controller_test.go index d91818103..20e39c49a 100644 --- a/controllers/machineinventory_controller_test.go +++ b/controllers/machineinventory_controller_test.go @@ -29,10 +29,13 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" elementalv1 "github.com/rancher/elemental-operator/api/v1beta1" "github.com/rancher/elemental-operator/pkg/test" + + apierrors "k8s.io/apimachinery/pkg/api/errors" ) var _ = Describe("reconcile machine inventory", func() { @@ -118,6 +121,32 @@ var _ = Describe("reconcile machine inventory", func() { Expect(cond.Status).To(Equal(metav1.ConditionTrue)) Expect(cond.Message).To(Equal("plan successfully applied")) }) + + It("should add finalizer if not exist", func() { + noFinalizerMI := &elementalv1.MachineInventory{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-inventory-no-finalizer", + Namespace: "default", + }, + } + Expect(cl.Create(ctx, noFinalizerMI)).To(Succeed()) + + _, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: mInventory.Namespace, + Name: mInventory.Name, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + Expect(cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, noFinalizerMI)).To(Succeed()) + + Expect(controllerutil.ContainsFinalizer(noFinalizerMI, elementalv1.MachineInventoryFinalizer)).To(BeTrue()) + Expect(test.CleanupAndWait(ctx, cl, noFinalizerMI)).To(Succeed()) + }) }) var _ = Describe("createPlanSecret", func() { @@ -379,6 +408,135 @@ var _ = Describe("updateInventoryWithPlanStatus", func() { }) +var _ = Describe("handle finalizer", func() { + var r *MachineInventoryReconciler + var mInventory *elementalv1.MachineInventory + var planSecret *corev1.Secret + + BeforeEach(func() { + r = &MachineInventoryReconciler{ + Client: cl, + } + + planSecret = &corev1.Secret{} + + mInventory = &elementalv1.MachineInventory{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Annotations: map[string]string{elementalv1.MachineInventoryResettableAnnotation: "true"}, + Finalizers: []string{elementalv1.MachineInventoryFinalizer}, + Name: "machine-inventory-suite-finalizer", + Namespace: "default", + }, + } + + // 1. Create initial MachineInventory + Expect(cl.Create(ctx, mInventory)).To(Succeed()) + // 2. Create initial plan Secret + _, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: mInventory.Namespace, + Name: mInventory.Name, + }, + }) + Expect(err).ToNot(HaveOccurred()) + // 3. Update meta.DeletionTimestamp + Expect(cl.Delete(ctx, mInventory)).To(Succeed()) + // 4. Update secret with reset plan + _, err = r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: mInventory.Namespace, + Name: mInventory.Name, + }, + }) + Expect(err).ToNot(HaveOccurred()) + // 5. Update MachineInventory plan status + _, err = r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: mInventory.Namespace, + Name: mInventory.Name, + }, + }) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should update secret with reset plan", func() { + Expect(cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, mInventory)).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, planSecret)).To(Succeed()) + + wantChecksum, wantPlan, err := r.newResetPlan(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Check Plan status + Expect(mInventory.Status.Plan.Checksum).To(Equal(wantChecksum)) + Expect(mInventory.Status.Plan.PlanSecretRef.Name).To(Equal(planSecret.Name)) + Expect(mInventory.Status.Plan.PlanSecretRef.Namespace).To(Equal(planSecret.Namespace)) + Expect(mInventory.Status.Plan.State).To(Equal(elementalv1.PlanState(""))) + + // Check MachineInventory status + Expect(mInventory.Status.Conditions[0].Type).To(Equal(elementalv1.ReadyCondition)) + Expect(mInventory.Status.Conditions[0].Reason).To(Equal(elementalv1.WaitingForPlanReason)) + Expect(mInventory.Status.Conditions[0].Status).To(Equal(metav1.ConditionFalse)) + Expect(mInventory.Status.Conditions[0].Message).To(Equal("waiting for plan to be applied")) + + // Check plan secret was updated + Expect(planSecret.Annotations[elementalv1.PlanTypeAnnotation]).To(Equal(elementalv1.PlanTypeReset)) + Expect(planSecret.Data["plan"]).To(Equal(wantPlan)) + Expect(planSecret.Data["applied-checksum"]).To(Equal([]byte(""))) + Expect(planSecret.Data["failed-checksum"]).To(Equal([]byte(""))) + + // Check we are holding on the MachineInventory (preventing actual deletion) + _, err = r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: mInventory.Namespace, + Name: mInventory.Name, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, mInventory)).To(Succeed()) + }) + + It("should remove finalizer on reset plan applied", func() { + // 6. Mark the reset plan as applied + Expect(cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, planSecret)).To(Succeed()) + planSecret.Data["applied-checksum"] = []byte("applied") + Expect(cl.Update(ctx, planSecret)).To(Succeed()) + + // 7. Trigger deletion + _, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: mInventory.Namespace, + Name: mInventory.Name, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + // Check MachineInventory was actually deleted + err = cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, mInventory) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + AfterEach(func() { + Expect(test.CleanupAndWait(ctx, cl, mInventory, planSecret)).To(Succeed()) + }) +}) + type machineInventoryFailingClient struct { client.Client } From f66ad7ca5044c968476e546ef5526ce4e291bf55 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 16:24:19 +0200 Subject: [PATCH 41/50] Improve test coverage --- controllers/machineinventory_controller.go | 2 + .../machineinventory_controller_test.go | 63 ++++++++++++++++--- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 172cd6ef3..1b7848a18 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -204,6 +204,8 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex return fmt.Errorf("getting plan secret: %w", err) } + fmt.Printf("------------> MachineInventory: %+v", mInventory) + fmt.Printf("------------> PlanSecret: %+v", planSecret) if !util.IsObjectOwned(&planSecret.ObjectMeta, mInventory.UID) { return fmt.Errorf("secret already exists and was not created by this controller") } diff --git a/controllers/machineinventory_controller_test.go b/controllers/machineinventory_controller_test.go index 20e39c49a..bbd7cc71e 100644 --- a/controllers/machineinventory_controller_test.go +++ b/controllers/machineinventory_controller_test.go @@ -418,8 +418,6 @@ var _ = Describe("handle finalizer", func() { Client: cl, } - planSecret = &corev1.Secret{} - mInventory = &elementalv1.MachineInventory{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &metav1.Time{Time: time.Now()}, @@ -430,6 +428,13 @@ var _ = Describe("handle finalizer", func() { }, } + planSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, + } + // 1. Create initial MachineInventory Expect(cl.Create(ctx, mInventory)).To(Succeed()) // 2. Create initial plan Secret @@ -462,14 +467,13 @@ var _ = Describe("handle finalizer", func() { It("should update secret with reset plan", func() { Expect(cl.Get(ctx, client.ObjectKey{ - Name: mInventory.Name, - Namespace: mInventory.Namespace, - }, mInventory)).To(Succeed()) - + Name: planSecret.Name, + Namespace: planSecret.Namespace, + }, planSecret)).To(Succeed()) Expect(cl.Get(ctx, client.ObjectKey{ Name: mInventory.Name, Namespace: mInventory.Namespace, - }, planSecret)).To(Succeed()) + }, mInventory)).To(Succeed()) wantChecksum, wantPlan, err := r.newResetPlan(ctx) Expect(err).ToNot(HaveOccurred()) @@ -515,7 +519,33 @@ var _ = Describe("handle finalizer", func() { planSecret.Data["applied-checksum"] = []byte("applied") Expect(cl.Update(ctx, planSecret)).To(Succeed()) - // 7. Trigger deletion + // 7. Trigger finalizer removal + _, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: mInventory.Namespace, + Name: mInventory.Name, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + // Check MachineInventory was actually deleted + err = cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, mInventory) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should delete by removing resettable annotation when up for deletion", func() { + // 6. Manually disable resettable annotation + Expect(cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, mInventory)).To(Succeed()) + mInventory.Annotations[elementalv1.MachineInventoryResettableAnnotation] = "false" + Expect(cl.Update(ctx, mInventory)).To(Succeed()) + + // 7. Trigger finalizer removal _, err := r.Reconcile(ctx, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: mInventory.Namespace, @@ -532,6 +562,23 @@ var _ = Describe("handle finalizer", func() { Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) + It("should delete by removing finalizer when up for deletion", func() { + // 6. Manually remove finalizer + Expect(cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, mInventory)).To(Succeed()) + controllerutil.RemoveFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) + Expect(cl.Update(ctx, mInventory)).To(Succeed()) + + // Check MachineInventory was actually deleted + err := cl.Get(ctx, client.ObjectKey{ + Name: mInventory.Name, + Namespace: mInventory.Namespace, + }, mInventory) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + AfterEach(func() { Expect(test.CleanupAndWait(ctx, cl, mInventory, planSecret)).To(Succeed()) }) From 0c8533261166addb09519295d8e036d6ada9044d Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Wed, 2 Aug 2023 16:28:00 +0200 Subject: [PATCH 42/50] Remove debug printfs --- controllers/machineinventory_controller.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 1b7848a18..172cd6ef3 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -204,8 +204,6 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex return fmt.Errorf("getting plan secret: %w", err) } - fmt.Printf("------------> MachineInventory: %+v", mInventory) - fmt.Printf("------------> PlanSecret: %+v", planSecret) if !util.IsObjectOwned(&planSecret.ObjectMeta, mInventory.UID) { return fmt.Errorf("secret already exists and was not created by this controller") } From b8a39201cc93913be7e49f1e3e92de0b90554ee3 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 11:24:08 +0200 Subject: [PATCH 43/50] Check for nil delete timestamp pointer --- controllers/machineinventory_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 172cd6ef3..57521b3e3 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -121,7 +121,7 @@ func (r *MachineInventoryReconciler) reconcile(ctx context.Context, mInventory * logger.Info("Reconciling machineinventory object") - if mInventory.GetDeletionTimestamp().IsZero() { + if mInventory.GetDeletionTimestamp() == nil || mInventory.GetDeletionTimestamp().IsZero() { // The object is not being deleted, so register the finalizer if !controllerutil.ContainsFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) { controllerutil.AddFinalizer(mInventory, elementalv1.MachineInventoryFinalizer) From 51ea23e262eb26042d77ed494f48c515119bb370 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 11:32:47 +0200 Subject: [PATCH 44/50] Do not try to recover from missing secret plan --- controllers/machineinventory_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 57521b3e3..4500ae02f 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -21,6 +21,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "time" @@ -192,8 +193,7 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex } if mInventory.Status.Plan == nil || mInventory.Status.Plan.PlanSecretRef == nil { - logger.V(log.DebugDepth).Info("Machine inventory plan reference not set yet. Creating new empty plan.") - return r.createPlanSecret(ctx, mInventory) // Recover from this unexpected state by creating a new empty plan secret + return errors.New("machine inventory plan secret does not exist") } planSecret := &corev1.Secret{} From 541b918a933892b7aeecbe23130a3b510a62b227 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 11:36:40 +0200 Subject: [PATCH 45/50] Do not refetch plan secret twice --- controllers/machineinventory_controller.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index 4500ae02f..bc94064df 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -212,7 +212,7 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex if !annotationFound || planType != elementalv1.PlanTypeReset { logger.V(log.DebugDepth).Info("Non reset plan type found. Updating it with new reset plan.") - return r.updatePlanSecretWithReset(ctx, mInventory) + return r.updatePlanSecretWithReset(ctx, mInventory, planSecret) } logger.V(log.DebugDepth).Info("Reset plan type found. Updating status and determine whether it was successfully applied.") @@ -227,19 +227,11 @@ func (r *MachineInventoryReconciler) reconcileResetPlanSecret(ctx context.Contex return nil } -func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Context, mInventory *elementalv1.MachineInventory) error { +func (r *MachineInventoryReconciler) updatePlanSecretWithReset(ctx context.Context, mInventory *elementalv1.MachineInventory, planSecret *corev1.Secret) error { logger := ctrl.LoggerFrom(ctx) logger.Info("Updating Secret with Reset plan") - planSecret := &corev1.Secret{} - if err := r.Get(ctx, types.NamespacedName{ - Namespace: mInventory.Namespace, - Name: mInventory.Name, - }, planSecret); err != nil { - return fmt.Errorf("getting plan secret: %w", err) - } - checksum, resetPlan, err := r.newResetPlan(ctx) if err != nil { return fmt.Errorf("getting new reset plan: %w", err) From 7e207aa44e8f345b65f9c57cd23e06cdd4df575b Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 11:43:06 +0200 Subject: [PATCH 46/50] Rename reset cloud config file to more appropriate name --- controllers/machineinventory_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/machineinventory_controller.go b/controllers/machineinventory_controller.go index bc94064df..3c8cba23f 100644 --- a/controllers/machineinventory_controller.go +++ b/controllers/machineinventory_controller.go @@ -52,7 +52,7 @@ import ( // Timeout to validate machine inventory adoption const adoptionTimeout = 5 -const LocalResetPlanPath = "/oem/reset-plan.yaml" +const LocalResetPlanPath = "/oem/reset-cloud-config.yaml" // MachineInventoryReconciler reconciles a MachineInventory object. type MachineInventoryReconciler struct { From 284a5387389cda2eed8e131bdcd4258c96c54259 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 12:09:42 +0200 Subject: [PATCH 47/50] Use default paths on installation --- cmd/register/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index f5ab47e98..99439670f 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -176,7 +176,7 @@ func initConfig(fs vfs.FS) error { log.EnableDebugLogging() } // If we are installing from a live environment, the default config path must be updated - if installation && (configPath == defaultConfigPath) { + if installation { configPath = defaultLiveConfigPath statePath = defaultLiveStatePath } From ee4489b24e91632de763ca68d99af708350b8c62 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 12:30:12 +0200 Subject: [PATCH 48/50] Return after install or reset for safety --- cmd/register/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/register/main.go b/cmd/register/main.go index 99439670f..600bad4ba 100644 --- a/cmd/register/main.go +++ b/cmd/register/main.go @@ -122,11 +122,15 @@ func newCommand(fs vfs.FS, client register.Client, stateHandler register.StateHa if err := installer.InstallElemental(cfg, registrationState); err != nil { return fmt.Errorf("installing elemental: %w", err) } + return nil } // Reset if reset { log.Info("Resetting Elemental") - return installer.ResetElemental(cfg, registrationState) + if err := installer.ResetElemental(cfg, registrationState); err != nil { + return fmt.Errorf("resetting elemental: %w", err) + } + return nil } return nil From a60afd06fa1617d93aa5ce149ff0dd9edf824b52 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 12:52:19 +0200 Subject: [PATCH 49/50] Drop unused constant --- pkg/install/install.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/install/install.go b/pkg/install/install.go index 9fed08b56..b184b8c67 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -38,7 +38,6 @@ import ( ) const ( - stateInstallFile = "/run/initramfs/cos-state/state.yaml" agentStateDir = "/var/lib/elemental/agent" agentConfDir = "/etc/rancher/elemental/agent" registrationConf = "/oem/registration/config.yaml" From b00495b49d4f3fc82ba614e5a1432ea8727b9b47 Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Thu, 3 Aug 2023 15:36:17 +0200 Subject: [PATCH 50/50] Add verify generate and copyright to mock files --- Makefile | 9 ++++++++- pkg/elementalcli/mocks/elementalcli.go | 18 ++++++++++++++++++ pkg/install/mocks/install.go | 18 ++++++++++++++++++ pkg/register/mocks/client.go | 18 ++++++++++++++++++ pkg/register/mocks/state.go | 18 ++++++++++++++++++ scripts/generate_mocks.sh | 8 ++++---- 6 files changed, 84 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 2b16cc803..083529996 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ build-manifests: $(KUSTOMIZE) generate $(MAKE) build-crds $(MAKE) build-rbac -ALL_VERIFY_CHECKS = manifests vendor +ALL_VERIFY_CHECKS = manifests vendor generate .PHONY: verify verify: $(addprefix verify-,$(ALL_VERIFY_CHECKS)) @@ -247,3 +247,10 @@ verify-vendor: vendor git diff; \ echo "generated files are out of date, run make generate"; exit 1; \ fi + +.PHONY: verify-generate +verify-generate: generate + @if !(git diff --quiet HEAD); then \ + git diff; \ + echo "generated files are out of date, run make generate"; exit 1; \ + fi diff --git a/pkg/elementalcli/mocks/elementalcli.go b/pkg/elementalcli/mocks/elementalcli.go index cb81b6058..36b00a57e 100644 --- a/pkg/elementalcli/mocks/elementalcli.go +++ b/pkg/elementalcli/mocks/elementalcli.go @@ -1,3 +1,21 @@ +// /* +// Copyright © 2022 - 2023 SUSE LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ +// +// + // Code generated by MockGen. DO NOT EDIT. // Source: github.com/rancher/elemental-operator/pkg/elementalcli (interfaces: Runner) diff --git a/pkg/install/mocks/install.go b/pkg/install/mocks/install.go index e6d89f787..81eb191ef 100644 --- a/pkg/install/mocks/install.go +++ b/pkg/install/mocks/install.go @@ -1,3 +1,21 @@ +// /* +// Copyright © 2022 - 2023 SUSE LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ +// +// + // Code generated by MockGen. DO NOT EDIT. // Source: github.com/rancher/elemental-operator/pkg/install (interfaces: Installer) diff --git a/pkg/register/mocks/client.go b/pkg/register/mocks/client.go index 71bf89086..a0b0f486b 100644 --- a/pkg/register/mocks/client.go +++ b/pkg/register/mocks/client.go @@ -1,3 +1,21 @@ +// /* +// Copyright © 2022 - 2023 SUSE LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ +// +// + // Code generated by MockGen. DO NOT EDIT. // Source: github.com/rancher/elemental-operator/pkg/register (interfaces: Client) diff --git a/pkg/register/mocks/state.go b/pkg/register/mocks/state.go index a16f2a8eb..cd59438bf 100644 --- a/pkg/register/mocks/state.go +++ b/pkg/register/mocks/state.go @@ -1,3 +1,21 @@ +// /* +// Copyright © 2022 - 2023 SUSE LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ +// +// + // Code generated by MockGen. DO NOT EDIT. // Source: github.com/rancher/elemental-operator/pkg/register (interfaces: StateHandler) diff --git a/scripts/generate_mocks.sh b/scripts/generate_mocks.sh index 42ee4e84e..c1671e18e 100755 --- a/scripts/generate_mocks.sh +++ b/scripts/generate_mocks.sh @@ -5,7 +5,7 @@ go install go.uber.org/mock/mockgen@latest # Always create mock files into a "mocks" subfolder to be ignored in test coverage. # See codecov.yml for more info -mockgen -destination=pkg/register/mocks/client.go -package=mocks github.com/rancher/elemental-operator/pkg/register Client -mockgen -destination=pkg/register/mocks/state.go -package=mocks github.com/rancher/elemental-operator/pkg/register StateHandler -mockgen -destination=pkg/install/mocks/install.go -package=mocks github.com/rancher/elemental-operator/pkg/install Installer -mockgen -destination=pkg/elementalcli/mocks/elementalcli.go -package=mocks github.com/rancher/elemental-operator/pkg/elementalcli Runner +mockgen -copyright_file=scripts/boilerplate.go.txt -destination=pkg/register/mocks/client.go -package=mocks github.com/rancher/elemental-operator/pkg/register Client +mockgen -copyright_file=scripts/boilerplate.go.txt -destination=pkg/register/mocks/state.go -package=mocks github.com/rancher/elemental-operator/pkg/register StateHandler +mockgen -copyright_file=scripts/boilerplate.go.txt -destination=pkg/install/mocks/install.go -package=mocks github.com/rancher/elemental-operator/pkg/install Installer +mockgen -copyright_file=scripts/boilerplate.go.txt -destination=pkg/elementalcli/mocks/elementalcli.go -package=mocks github.com/rancher/elemental-operator/pkg/elementalcli Runner