[teleport-update] Uninstall subcommand (#49341)

* Uninstall

* tests

* comment

* Short-circuit link package on pinned

* log

* move error

* Update lib/autoupdate/agent/process.go

Co-authored-by: Hugo Shaka <hugo.hervieux@goteleport.com>

* Update lib/autoupdate/agent/process.go

Co-authored-by: Hugo Shaka <hugo.hervieux@goteleport.com>

* Update lib/autoupdate/agent/process.go

Co-authored-by: Hugo Shaka <hugo.hervieux@goteleport.com>

* Update lib/autoupdate/agent/process.go

Co-authored-by: Hugo Shaka <hugo.hervieux@goteleport.com>

* fix

---------

Co-authored-by: Hugo Shaka <hugo.hervieux@goteleport.com>

lib/autoupdate/agent/installer.go

@@ -575,7 +575,7 @@ func (li *LocalInstaller) forceLinks(ctx context.Context, binDir, svcDir string)
 		linked++
 	}
 	if linked == 0 {
-		return revert, trace.Errorf("no binaries available to link")
+		return revert, trace.Wrap(ErrNoBinaries)
 	}
 
 	// create systemd service file
@@ -784,7 +784,7 @@ func (li *LocalInstaller) tryLinks(ctx context.Context, binDir, svcDir string) e
 	}
 	// bail if no binaries can be linked
 	if linked == 0 {
-		return trace.Errorf("no binaries available to link")
+		return trace.Wrap(ErrNoBinaries)
 	}
 
 	// link binaries that are missing links

lib/autoupdate/agent/process.go

@@ -111,7 +111,11 @@ func (s SystemdService) Reload(ctx context.Context) error {
 	}
 	if initPID != 0 {
 		s.Log.InfoContext(ctx, "Monitoring PID file to detect crashes.", unitKey, s.ServiceName)
-		return trace.Wrap(s.monitor(ctx, initPID))
+		err := s.monitor(ctx, initPID)
+		if errors.Is(err, context.DeadlineExceeded) {
+			return trace.Errorf("timed out while waiting for process to start")
+		}
+		return trace.Wrap(err)
 	}
 	return nil
 }
@@ -265,14 +269,49 @@ func (s SystemdService) Enable(ctx context.Context, now bool) error {
 	if now {
 		args = append(args, "--now")
 	}
-	code := s.systemctl(ctx, slog.LevelError, args...)
+	code := s.systemctl(ctx, slog.LevelInfo, args...)
 	if code != 0 {
 		return trace.Errorf("unable to enable systemd service")
 	}
 	s.Log.InfoContext(ctx, "Service enabled.", unitKey, s.ServiceName)
 	return nil
 }
 
+// Disable the systemd service.
+func (s SystemdService) Disable(ctx context.Context) error {
+	if err := s.checkSystem(ctx); err != nil {
+		return trace.Wrap(err)
+	}
+	code := s.systemctl(ctx, slog.LevelInfo, "disable", s.ServiceName)
+	if code != 0 {
+		return trace.Errorf("unable to disable systemd service")
+	}
+	s.Log.InfoContext(ctx, "Systemd service disabled.", unitKey, s.ServiceName)
+	return nil
+}
+
+// IsEnabled returns true if the service is enabled, or if it's disabled but still active.
+func (s SystemdService) IsEnabled(ctx context.Context) (bool, error) {
+	if err := s.checkSystem(ctx); err != nil {
+		return false, trace.Wrap(err)
+	}
+	code := s.systemctl(ctx, slog.LevelDebug, "is-enabled", "--quiet", s.ServiceName)
+	switch {
+	case code < 0:
+		return false, trace.Errorf("unable to determine if systemd service %q is enabled", s.ServiceName)
+	case code == 0:
+		return true, nil
+	}
+	code = s.systemctl(ctx, slog.LevelDebug, "is-active", "--quiet", s.ServiceName)
+	switch {
+	case code < 0:
+		return false, trace.Errorf("unable to determine if systemd service %q is active", s.ServiceName)
+	case code == 0:
+		return true, nil
+	}
+	return false, nil
+}
+
 // checkSystem returns an error if the system is not compatible with this process manager.
 func (s SystemdService) checkSystem(ctx context.Context) error {
 	_, err := os.Stat("/run/systemd/system")

lib/autoupdate/agent/setup.go

@@ -20,6 +20,8 @@ package agent
 
 import (
 	"context"
+	"errors"
+	"io/fs"
 	"log/slog"
 	"os"
 	"path/filepath"
@@ -72,6 +74,32 @@ func Setup(ctx context.Context, log *slog.Logger, linkDir, dataDir string) error
 	return nil
 }
 
+// Teardown removes all traces of the auto-updater, including its configuration.
+func Teardown(ctx context.Context, log *slog.Logger, linkDir, dataDir string) error {
+	svc := &SystemdService{
+		ServiceName: "teleport-update.timer",
+		Log:         log,
+	}
+	if err := svc.Disable(ctx); err != nil {
+		return trace.Errorf("failed to disable teleport-update systemd timer: %w", err)
+	}
+	servicePath := filepath.Join(linkDir, serviceDir, updateServiceName)
+	if err := os.Remove(servicePath); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return trace.Errorf("failed to remove teleport-update systemd service: %w", err)
+	}
+	timerPath := filepath.Join(linkDir, serviceDir, updateTimerName)
+	if err := os.Remove(timerPath); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return trace.Errorf("failed to remove teleport-update systemd timer: %w", err)
+	}
+	if err := svc.Sync(ctx); err != nil {
+		return trace.Errorf("failed to sync systemd config: %w", err)
+	}
+	if err := os.RemoveAll(filepath.Join(dataDir, VersionsDirName)); err != nil {
+		return trace.Errorf("failed to remove versions directory: %w", err)
+	}
+	return nil
+}
+
 func writeConfigFiles(linkDir, dataDir string) error {
 	servicePath := filepath.Join(linkDir, serviceDir, updateServiceName)
 	err := writeTemplate(servicePath, updateServiceTemplate, linkDir, dataDir)

lib/autoupdate/agent/testdata/TestUpdater_Install/no_need_to_reload.golden

@@ -0,0 +1,9 @@
+version: v1
+kind: update_config
+spec:
+    proxy: localhost
+    enabled: false
+    pinned: false
+status:
+    active_version: 16.3.0
+    backup_version: ""

lib/autoupdate/agent/testdata/TestUpdater_Install/no_systemd.golden

@@ -0,0 +1,9 @@
+version: v1
+kind: update_config
+spec:
+    proxy: localhost
+    enabled: false
+    pinned: false
+status:
+    active_version: 16.3.0
+    backup_version: ""

lib/autoupdate/agent/updater.go

@@ -150,6 +150,9 @@ func NewLocalUpdater(cfg LocalUpdaterConfig) (*Updater, error) {
 		Revert: func(ctx context.Context) error {
 			return trace.Wrap(Setup(ctx, cfg.Log, cfg.LinkDir, cfg.DataDir))
 		},
+		Teardown: func(ctx context.Context) error {
+			return trace.Wrap(Teardown(ctx, cfg.Log, cfg.LinkDir, cfg.DataDir))
+		},
 	}, nil
 }
 
@@ -191,6 +194,8 @@ type Updater struct {
 	Setup func(ctx context.Context) error
 	// Revert installs the Teleport updater service using the running installation.
 	Revert func(ctx context.Context) error
+	// Teardown removes all traces of the updater and all managed installations.
+	Teardown func(ctx context.Context) error
 }
 
 // Installer provides an API for installing Teleport agents.
@@ -214,8 +219,11 @@ type Installer interface {
 	// Unlike LinkSystem, TryLinkSystem will fail if existing links to other locations are present.
 	// TryLinkSystem must be idempotent.
 	TryLinkSystem(ctx context.Context) error
+	// Unlink unlinks the specified version of Teleport from the linking locations.
+	// Unlink must be idempotent.
+	Unlink(ctx context.Context, version string) error
 	// UnlinkSystem unlinks the system (package) installation of Teleport from the linking locations.
-	// TryLinkSystem must be idempotent.
+	// UnlinkSystem must be idempotent.
 	UnlinkSystem(ctx context.Context) error
 	// List the installed versions of Teleport.
 	List(ctx context.Context) (versions []string, err error)
@@ -232,6 +240,8 @@ var (
 	ErrNotNeeded = errors.New("not needed")
 	// ErrNotSupported is returned when the operation is not supported on the platform.
 	ErrNotSupported = errors.New("not supported on this platform")
+	// ErrNoBinaries is returned when no binaries are available to be linked.
+	ErrNoBinaries = errors.New("no binaries available to link")
 )
 
 const (
@@ -254,6 +264,10 @@ type Process interface {
 	// If the type implementing Process does not support the system process manager,
 	// Sync must return ErrNotSupported.
 	Sync(ctx context.Context) error
+	// IsEnabled must return true if the Process is running or is configured to run.
+	// If the type implementing Process does not support the system process manager,
+	// Sync must return ErrNotSupported.
+	IsEnabled(ctx context.Context) (bool, error)
 }
 
 // TODO(sclevine): add support for need_restart and selinux config
@@ -324,6 +338,111 @@ func (u *Updater) Install(ctx context.Context, override OverrideConfig) error {
 	return nil
 }
 
+// Remove removes everything created by the updater.
+// Before attempting this, Remove attempts to gracefully recover the system-packaged version of Teleport (if present).
+// This function is idempotent.
+func (u *Updater) Remove(ctx context.Context) error {
+	cfg, err := readConfig(u.ConfigPath)
+	if err != nil {
+		return trace.Errorf("failed to read %s: %w", updateConfigName, err)
+	}
+	if err := validateConfigSpec(&cfg.Spec, OverrideConfig{}); err != nil {
+		return trace.Wrap(err)
+	}
+	activeVersion := cfg.Status.ActiveVersion
+	if activeVersion == "" {
+		u.Log.InfoContext(ctx, "No installation of Teleport managed by the updater. Removing updater configuration.")
+		if err := u.Teardown(ctx); err != nil {
+			return trace.Wrap(err)
+		}
+		u.Log.InfoContext(ctx, "Automatic update configuration for Teleport successfully uninstalled.")
+		return nil
+	}
+
+	revert, err := u.Installer.LinkSystem(ctx)
+	if errors.Is(err, ErrNoBinaries) {
+		u.Log.InfoContext(ctx, "Updater-managed installation of Teleport detected. Attempting to unlink and remove.")
+		ok, err := u.Process.IsEnabled(ctx)
+		if err != nil && !errors.Is(err, ErrNotSupported) {
+			return trace.Wrap(err)
+		}
+		if ok {
+			return trace.Errorf("refusing to remove active installation of Teleport, please disable Teleport first")
+		}
+		if err := u.Installer.Unlink(ctx, activeVersion); err != nil {
+			return trace.Wrap(err)
+		}
+		u.Log.InfoContext(ctx, "Teleport uninstalled.", "version", activeVersion)
+		if err := u.Teardown(ctx); err != nil {
+			return trace.Wrap(err)
+		}
+		u.Log.InfoContext(ctx, "Automatic update configuration for Teleport successfully uninstalled.")
+		return nil
+	}
+	if err != nil {
+		return trace.Errorf("failed to link: %w", err)
+	}
+
+	u.Log.InfoContext(ctx, "Updater-managed installation of Teleport detected. Restoring packaged version of Teleport before removing.")
+
+	revertConfig := func(ctx context.Context) bool {
+		if ok := revert(ctx); !ok {
+			u.Log.ErrorContext(ctx, "Failed to revert Teleport symlinks. Installation likely broken.")
+			return false
+		}
+		if err := u.Process.Sync(ctx); err != nil {
+			u.Log.ErrorContext(ctx, "Failed to revert systemd configuration after failed restart.", errorKey, err)
+			return false
+		}
+		return true
+	}
+
+	// Sync systemd.
+
+	err = u.Process.Sync(ctx)
+	if errors.Is(err, ErrNotSupported) {
+		u.Log.WarnContext(ctx, "Not syncing systemd configuration because systemd is not running.")
+	} else if errors.Is(err, context.Canceled) {
+		return trace.Errorf("sync canceled")
+	} else if err != nil {
+		// If sync fails, we may have left the host in a bad state, so we revert linking and re-Sync.
+		u.Log.ErrorContext(ctx, "Reverting symlinks due to invalid configuration.")
+		if ok := revertConfig(ctx); ok {
+			u.Log.WarnContext(ctx, "Teleport updater encountered a configuration error and successfully reverted the installation.")
+		}
+		return trace.Errorf("failed to validate configuration for system package version of Teleport: %w", err)
+	}
+
+	// Restart Teleport.
+
+	u.Log.InfoContext(ctx, "Teleport package successfully restored.")
+	err = u.Process.Reload(ctx)
+	if errors.Is(err, context.Canceled) {
+		return trace.Errorf("reload canceled")
+	}
+	if err != nil &&
+		!errors.Is(err, ErrNotNeeded) && // no output if restart not needed
+		!errors.Is(err, ErrNotSupported) { // already logged above for Sync
+
+		// If reloading Teleport at the new version fails, revert and reload.
+		u.Log.ErrorContext(ctx, "Reverting symlinks due to failed restart.")
+		if ok := revertConfig(ctx); ok {
+			if err := u.Process.Reload(ctx); err != nil && !errors.Is(err, ErrNotNeeded) {
+				u.Log.ErrorContext(ctx, "Failed to reload Teleport after reverting. Installation likely broken.", errorKey, err)
+			} else {
+				u.Log.WarnContext(ctx, "Teleport updater detected an error with the new installation and successfully reverted it.")
+			}
+		}
+		return trace.Errorf("failed to start system package version of Teleport: %w", err)
+	}
+	u.Log.InfoContext(ctx, "Auto-updating Teleport removed and replaced by Teleport packaged.", "version", activeVersion)
+	if err := u.Teardown(ctx); err != nil {
+		return trace.Wrap(err)
+	}
+	u.Log.InfoContext(ctx, "Auto-update configuration for Teleport successfully uninstalled.")
+	return nil
+}
+
 // Status returns all available local and remote fields related to agent auto-updates.
 func (u *Updater) Status(ctx context.Context) (Status, error) {
 	var out Status
@@ -585,7 +704,7 @@ func (u *Updater) update(ctx context.Context, cfg *UpdateConfig, targetVersion s
 				if err := u.Process.Reload(ctx); err != nil && !errors.Is(err, ErrNotNeeded) {
 					u.Log.ErrorContext(ctx, "Failed to reload Teleport after reverting. Installation likely broken.", errorKey, err)
 				} else {
-					u.Log.WarnContext(ctx, "Teleport updater encountered a configuration error and successfully reverted the installation.")
+					u.Log.WarnContext(ctx, "Teleport updater detected an error with the new installation and successfully reverted it.")
 				}
 			}
 			return trace.Errorf("failed to start new version %q of Teleport: %w", targetVersion, err)
@@ -624,14 +743,18 @@ func (u *Updater) LinkPackage(ctx context.Context) error {
 	}
 	activeVersion := cfg.Status.ActiveVersion
 	if cfg.Spec.Enabled {
-		u.Log.InfoContext(ctx, "Automatic updates enabled. Skipping system package link.", activeVersionKey, activeVersion)
+		u.Log.InfoContext(ctx, "Automatic updates is enabled. Skipping system package link.", activeVersionKey, activeVersion)
+		return nil
+	}
+	if cfg.Spec.Pinned {
+		u.Log.InfoContext(ctx, "Automatic update version is pinned. Skipping system package link.", activeVersionKey, activeVersion)
 		return nil
 	}
 	// If an active version is set, but auto-updates is disabled, try to link the system installation in case the config is stale.
 	// If any links are present, this will return ErrLinked and not create any system links.
 	// This state is important to log as a warning,
 	if err := u.Installer.TryLinkSystem(ctx); errors.Is(err, ErrLinked) {
-		u.Log.WarnContext(ctx, "Automatic updates disabled, but a non-package version of Teleport is linked.", activeVersionKey, activeVersion)
+		u.Log.WarnContext(ctx, "Automatic updates is disabled, but a non-package version of Teleport is linked.", activeVersionKey, activeVersion)
 		return nil
 	} else if err != nil {
 		return trace.Errorf("failed to link system package installation: %w", err)