diff --git a/Dockerfile b/Dockerfile index 3eff0e4..d5b7245 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # syntax=docker/dockerfile:1 FROM golang:1.21 +# Build a static binary +ENV CGO_ENABLED=0 + # Set destination for COPY WORKDIR /app diff --git a/README.md b/README.md index f0ffcd5..5003d10 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,15 @@ # Specific Architecture ./build/docker.sh --architecture arm64 ls -la -... ebs-bootstrap_linux-arm64 +... ebs-bootstrap-linux-aarch64 # All Architectures ./build/docker.sh ls -la -... ebs-bootstrap_linux-arm64 -... ebs-bootstrap_linux-x86_64 +... ebs-bootstrap-linux-aarch64 +... ebs-bootstrap-linux-x86_64 ``` + ## Recommended Setup ### `systemd` diff --git a/assets/uml.drawio b/assets/uml.drawio new file mode 100644 index 0000000..7d1d69a --- /dev/null +++ b/assets/uml.drawio @@ -0,0 +1,276 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/uml.png b/assets/uml.png new file mode 100644 index 0000000..9884ef1 Binary files /dev/null and b/assets/uml.png differ diff --git a/cmd/ebs-bootstrap.go b/cmd/ebs-bootstrap.go index 21f0d8e..cd3a3b4 100644 --- a/cmd/ebs-bootstrap.go +++ b/cmd/ebs-bootstrap.go @@ -1,44 +1,90 @@ package main import ( - "os" "log" - "ebs-bootstrap/internal/config" - "ebs-bootstrap/internal/service" - "ebs-bootstrap/internal/utils" - "ebs-bootstrap/internal/state" + "os" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/layer" + "github.com/reecetech/ebs-bootstrap/internal/service" + "github.com/reecetech/ebs-bootstrap/internal/utils" ) func main() { - // Disable Timetamp log.SetFlags(0) - e := utils.NewExecRunner() - ds := &service.LinuxDeviceService{Runner: e} - ns := &service.AwsNVMeService{} - fs := &service.UnixFileService{} - dts := &service.EbsDeviceTranslator{DeviceService: ds, NVMeService: ns} - dt, err := dts.GetTranslator() - if err != nil { - log.Fatal(err) + // Services + erf := utils.NewExecRunnerFactory() + ufs := service.NewUnixFileService() + lds := service.NewLinuxDeviceService(erf) + uos := service.NewUnixOwnerService() + ans := service.NewAwsNitroNVMeService() + fssf := service.NewLinuxFileSystemServiceFactory(erf) + + // Warnings + warnings(uos) + + // Config + Flags + c, err := config.New(os.Args) + checkError(err) + + // Service + Config Consumers + db := backend.NewLinuxDeviceBackend(lds, fssf) + fb := backend.NewLinuxFileBackend(ufs) + ub := backend.NewLinuxOwnerBackend(uos) + dmb := backend.NewLinuxDeviceMetricsBackend(lds, fssf) + dae := action.NewDefaultActionExecutor(c) + + // Modify Config + modifiers := []config.Modifier{ + config.NewAwsNVMeDriverModifier(ans, lds), + } + for _, m := range modifiers { + checkError(m.Modify(c)) } - config, err := config.New(os.Args, dt, fs) + + // Validate Config + validators := []config.Validator{ + config.NewFileSystemValidator(), + config.NewModeValidator(), + config.NewResizeThresholdValidator(), + config.NewDeviceValidator(lds), + config.NewMountPointValidator(), + config.NewMountOptionsValidator(), + config.NewOwnerValidator(uos), + } + for _, v := range validators { + checkError(v.Validate(c)) + } + + // Layers + le := layer.NewExponentialBackoffLayerExecutor(c, dae) + layers := []layer.Layer{ + layer.NewFormatDeviceLayer(db), + layer.NewLabelDeviceLayer(db), + layer.NewCreateDirectoryLayer(db, fb), + layer.NewMountDeviceLayer(db, fb), + layer.NewResizeDeviceLayer(db, dmb), + layer.NewChangeOwnerLayer(ub, fb), + layer.NewChangePermissionsLayer(fb), + } + checkError(le.Execute(layers)) +} + +func checkError(err error) { if err != nil { log.Fatal(err) } +} - for name, device := range config.Devices { - d, err := state.NewDevice(name, ds, fs) - if err != nil { - log.Fatal(err) - } - err = d.Diff(config) - if err == nil { - log.Printf("🟢 %s: No changes detected", name) - continue - } - if device.Mode == "healthcheck" { - log.Fatal(err) - } +func warnings(us service.OwnerService) { + cu, err := us.GetCurrentUser() + if err != nil { + return + } + if cu.Uid != 0 { + log.Println("🚧 Not running as root user. Operations that query and modify block devices will likely be restricted") } } diff --git a/configs/ebs-bootstrap.yml b/configs/ebs-bootstrap.yml deleted file mode 100644 index e6df7ff..0000000 --- a/configs/ebs-bootstrap.yml +++ /dev/null @@ -1,11 +0,0 @@ -global: - mode: healthcheck -devices: - /dev/xvdf: - fs: "xfs" - mount_point: "/mnt/app" - owner: 1000 - group: 1000 - permissions: 755 - label: "external-vol" - mode: healthcheck diff --git a/configs/ubuntu.yml b/configs/ubuntu.yml new file mode 100644 index 0000000..6ef2b2e --- /dev/null +++ b/configs/ubuntu.yml @@ -0,0 +1,9 @@ +devices: + /dev/vdb: + fs: ext4 + label: external-vol + mountPoint: /mnt/app + group: ubuntu + user: ubuntu + permissions: 0644 + resizeFs: true \ No newline at end of file diff --git a/go.mod b/go.mod index 88788c8..662b26d 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,10 @@ -module ebs-bootstrap +module github.com/reecetech/ebs-bootstrap go 1.21 require gopkg.in/yaml.v2 v2.4.0 -require github.com/google/go-cmp v0.6.0 // indirect +require ( + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/google/go-cmp v0.6.0 // indirect +) diff --git a/go.sum b/go.sum index 75c0544..3796867 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/action/action.go b/internal/action/action.go new file mode 100644 index 0000000..4f3b6f5 --- /dev/null +++ b/internal/action/action.go @@ -0,0 +1,86 @@ +package action + +import ( + "fmt" + "log" + "strings" + + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" +) + +type Action interface { + Execute() error + Success() string + Prompt() string + Refuse() string + GetMode() model.Mode + SetMode(mode model.Mode) Action +} + +type ActionExecutor interface { + Execute(actions []Action) error +} + +type DefaultActionExecutor struct { + config *config.Config + read func(buffer *string) error +} + +func NewDefaultActionExecutor(c *config.Config) *DefaultActionExecutor { + return &DefaultActionExecutor{ + config: c, + read: func(buffer *string) error { + _, err := fmt.Scanln(buffer) + return err + }, + } +} + +func (dae *DefaultActionExecutor) Execute(actions []Action) error { + for _, a := range actions { + err := dae.execute(a) + if err != nil { + return err + } + } + return nil +} + +func (dae *DefaultActionExecutor) execute(action Action) error { + switch action.GetMode() { + case model.Force: + break + case model.Prompt: + if !dae.shouldProceed(action) { + return fmt.Errorf("🔴 Action rejected. %s", action.Refuse()) + } + case model.Healthcheck: + return fmt.Errorf("🔴 Healthcheck mode enabled. %s", action.Refuse()) + default: + return fmt.Errorf("🔴 Unexpected mode detected. %s", action.Refuse()) + } + + if err := action.Execute(); err != nil { + return err + } + log.Printf("⭐ %s", action.Success()) + return nil +} + +func (dae *DefaultActionExecutor) shouldProceed(action Action) bool { + prompt := action.Prompt() + + fmt.Printf("🟣 %s? (y/n): ", prompt) + var response string + err := dae.read(&response) + if err != nil { + return false + } + + response = strings.ToLower(response) + if response == "y" || response == "yes" { + return true + } + return false +} diff --git a/internal/action/file.go b/internal/action/file.go new file mode 100644 index 0000000..a58003e --- /dev/null +++ b/internal/action/file.go @@ -0,0 +1,131 @@ +package action + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type CreateDirectoryAction struct { + path string + mode model.Mode + fileService service.FileService +} + +func NewCreateDirectoryAction(p string, fs service.FileService) *CreateDirectoryAction { + return &CreateDirectoryAction{ + path: p, + mode: model.Empty, + fileService: fs, + } +} + +func (a *CreateDirectoryAction) Execute() error { + return a.fileService.CreateDirectory(a.path) +} + +func (a *CreateDirectoryAction) GetMode() model.Mode { + return a.mode +} + +func (a *CreateDirectoryAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *CreateDirectoryAction) Prompt() string { + return fmt.Sprintf("Would you like to recursively create directory %s", a.path) +} + +func (a *CreateDirectoryAction) Refuse() string { + return fmt.Sprintf("Refused to create directory %s", a.path) +} + +func (a *CreateDirectoryAction) Success() string { + return fmt.Sprintf("Successfully created directory %s", a.path) +} + +type ChangeOwnerAction struct { + path string + uid int + gid int + mode model.Mode + fileService service.FileService +} + +func NewChangeOwnerAction(p string, uid int, gid int, fs service.FileService) *ChangeOwnerAction { + return &ChangeOwnerAction{ + path: p, + uid: uid, + gid: gid, + mode: model.Empty, + fileService: fs, + } +} + +func (a *ChangeOwnerAction) Execute() error { + return a.fileService.ChangeOwner(a.path, a.uid, a.gid) +} + +func (a *ChangeOwnerAction) GetMode() model.Mode { + return a.mode +} + +func (a *ChangeOwnerAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *ChangeOwnerAction) Prompt() string { + return fmt.Sprintf("Would you like to change ownership (%d:%d) of %s", a.uid, a.gid, a.path) +} + +func (a *ChangeOwnerAction) Refuse() string { + return fmt.Sprintf("Refused to to change ownership (%d:%d) of %s", a.uid, a.gid, a.path) +} + +func (a *ChangeOwnerAction) Success() string { + return fmt.Sprintf("Successfully changed ownership (%d:%d) of %s", a.uid, a.gid, a.path) +} + +type ChangePermissionsAction struct { + path string + perms model.FilePermissions + mode model.Mode + fileService service.FileService +} + +func NewChangePermissions(p string, perms model.FilePermissions, fs service.FileService) *ChangePermissionsAction { + return &ChangePermissionsAction{ + path: p, + perms: perms, + mode: model.Empty, + fileService: fs, + } +} + +func (a *ChangePermissionsAction) Execute() error { + return a.fileService.ChangePermissions(a.path, a.perms) +} + +func (a *ChangePermissionsAction) GetMode() model.Mode { + return a.mode +} + +func (a *ChangePermissionsAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *ChangePermissionsAction) Prompt() string { + return fmt.Sprintf("Would you like to change permissions of %s to %#o", a.path, a.perms) +} + +func (a *ChangePermissionsAction) Refuse() string { + return fmt.Sprintf("Refused to to change permissions of %s to %#o", a.path, a.perms) +} + +func (a *ChangePermissionsAction) Success() string { + return fmt.Sprintf("Successfully change permissions of %s to %#o", a.path, a.perms) +} diff --git a/internal/action/format.go b/internal/action/format.go new file mode 100644 index 0000000..185e137 --- /dev/null +++ b/internal/action/format.go @@ -0,0 +1,47 @@ +package action + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type FormatDeviceAction struct { + device string + fileSystemService service.FileSystemService + mode model.Mode +} + +func NewFormatDeviceAction(d string, fileSystemService service.FileSystemService) *FormatDeviceAction { + return &FormatDeviceAction{ + device: d, + fileSystemService: fileSystemService, + mode: model.Empty, + } +} + +func (a *FormatDeviceAction) Execute() error { + return a.fileSystemService.Format(a.device) +} + +func (a *FormatDeviceAction) GetMode() model.Mode { + return a.mode +} + +func (a *FormatDeviceAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *FormatDeviceAction) Prompt() string { + return fmt.Sprintf("Would you like to format %s to %s", a.device, a.fileSystemService.GetFileSystem()) +} + +func (a *FormatDeviceAction) Refuse() string { + return fmt.Sprintf("Refused to format to %s", a.fileSystemService.GetFileSystem()) +} + +func (a *FormatDeviceAction) Success() string { + return fmt.Sprintf("Successfully formated to %s", a.fileSystemService.GetFileSystem()) +} diff --git a/internal/action/label.go b/internal/action/label.go new file mode 100644 index 0000000..088dbc4 --- /dev/null +++ b/internal/action/label.go @@ -0,0 +1,49 @@ +package action + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type LabelDeviceAction struct { + device string + label string + fileSystemService service.FileSystemService + mode model.Mode +} + +func NewLabelDeviceAction(d string, label string, fileSystemService service.FileSystemService) *LabelDeviceAction { + return &LabelDeviceAction{ + device: d, + label: label, + fileSystemService: fileSystemService, + mode: model.Empty, + } +} + +func (a *LabelDeviceAction) Execute() error { + return a.fileSystemService.Label(a.device, a.label) +} + +func (a *LabelDeviceAction) GetMode() model.Mode { + return a.mode +} + +func (a *LabelDeviceAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *LabelDeviceAction) Prompt() string { + return fmt.Sprintf("Would you like to label device %s to %s", a.device, a.label) +} + +func (a *LabelDeviceAction) Refuse() string { + return fmt.Sprintf("Refused to label to %s", a.label) +} + +func (a *LabelDeviceAction) Success() string { + return fmt.Sprintf("Successfully labelled to %s", a.label) +} diff --git a/internal/action/mount.go b/internal/action/mount.go new file mode 100644 index 0000000..c5b6706 --- /dev/null +++ b/internal/action/mount.go @@ -0,0 +1,99 @@ +package action + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" + "github.com/reecetech/ebs-bootstrap/internal/utils" +) + +type MountDeviceAction struct { + source string + target string + fileSystem model.FileSystem + options model.MountOptions + deviceService service.DeviceService + mode model.Mode +} + +func NewMountDeviceAction(source string, target string, fileSystem model.FileSystem, options model.MountOptions, deviceService service.DeviceService) *MountDeviceAction { + return &MountDeviceAction{ + source: source, + target: target, + fileSystem: fileSystem, + options: options, + deviceService: deviceService, + mode: model.Empty, + } +} + +func (a *MountDeviceAction) Execute() error { + return a.deviceService.Mount(a.source, a.target, a.fileSystem, a.options) +} + +func (a *MountDeviceAction) GetMode() model.Mode { + return a.mode +} + +func (a *MountDeviceAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *MountDeviceAction) Prompt() string { + return fmt.Sprintf("Would you like to mount %s to %s (%s)", a.source, a.target, a.options) +} + +func (a *MountDeviceAction) Refuse() string { + return fmt.Sprintf("Refused to mount %s to %s (%s)", a.source, a.target, a.options) +} + +func (a *MountDeviceAction) Success() string { + return fmt.Sprintf("Successfully mounted %s to %s (%s)", a.source, a.target, a.options) +} + +type UnmountDeviceAction struct { + source string + target string + deviceService service.DeviceService + mode model.Mode +} + +func NewUnmountDeviceAction(source string, target string, deviceService service.DeviceService) *UnmountDeviceAction { + return &UnmountDeviceAction{ + source: source, + target: target, + deviceService: deviceService, + mode: model.Empty, + } +} + +func (a *UnmountDeviceAction) Execute() error { + return a.deviceService.Umount(a.source, a.target) +} + +func (a *UnmountDeviceAction) Preflight(rc *utils.ExecRunnerFactory) error { + return nil +} + +func (a *UnmountDeviceAction) GetMode() model.Mode { + return a.mode +} + +func (a *UnmountDeviceAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *UnmountDeviceAction) Prompt() string { + return fmt.Sprintf("Would you like to unmount %s from %s", a.source, a.target) +} + +func (a *UnmountDeviceAction) Refuse() string { + return fmt.Sprintf("Refused to unmount %s from %s", a.source, a.target) +} + +func (a *UnmountDeviceAction) Success() string { + return fmt.Sprintf("Successfully unmounted %s from %s", a.source, a.target) +} diff --git a/internal/action/resize.go b/internal/action/resize.go new file mode 100644 index 0000000..fcdbdfd --- /dev/null +++ b/internal/action/resize.go @@ -0,0 +1,53 @@ +package action + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type ResizeDeviceAction struct { + device string + target string + fileSystemService service.FileSystemService + mode model.Mode +} + +func NewResizeDeviceAction(d string, target string, fileSystemService service.FileSystemService) *ResizeDeviceAction { + return &ResizeDeviceAction{ + device: d, + target: target, + fileSystemService: fileSystemService, + mode: model.Empty, + } +} + +func (a *ResizeDeviceAction) Execute() error { + return a.fileSystemService.Resize(a.target) +} + +func (a *ResizeDeviceAction) GetMode() model.Mode { + return a.mode +} + +func (a *ResizeDeviceAction) SetMode(mode model.Mode) Action { + a.mode = mode + return a +} + +func (a *ResizeDeviceAction) Prompt() string { + return fmt.Sprintf("Would you like to resize the %s file system of %s", a.fileSystemService.GetFileSystem(), a.device) +} + +func (a *ResizeDeviceAction) Refuse() string { + return fmt.Sprintf("Refused to resize the %s file system of %s", a.fileSystemService.GetFileSystem(), a.device) +} + +func (a *ResizeDeviceAction) Success() string { + return fmt.Sprintf("Successfully resized the %s file system of %s", a.fileSystemService.GetFileSystem(), a.device) +} + +func (a *ResizeDeviceAction) Warning() string { + return "" +} diff --git a/internal/backend/device.go b/internal/backend/device.go new file mode 100644 index 0000000..3857743 --- /dev/null +++ b/internal/backend/device.go @@ -0,0 +1,122 @@ +package backend + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type DeviceBackend interface { + GetBlockDevice(device string) (*model.BlockDevice, error) + Label(bd *model.BlockDevice, label string) ([]action.Action, error) + Resize(bd *model.BlockDevice) (action.Action, error) + Format(bd *model.BlockDevice, fileSystem model.FileSystem) (action.Action, error) + Mount(bd *model.BlockDevice, target string, options model.MountOptions) action.Action + Remount(bd *model.BlockDevice, target string, options model.MountOptions) action.Action + Umount(bd *model.BlockDevice) action.Action + From(config *config.Config) error +} + +type LinuxDeviceBackend struct { + blockDevices map[string]*model.BlockDevice + deviceService service.DeviceService + fileSystemServiceFactory service.FileSystemServiceFactory +} + +func NewLinuxDeviceBackend(ds service.DeviceService, fssf service.FileSystemServiceFactory) *LinuxDeviceBackend { + return &LinuxDeviceBackend{ + blockDevices: map[string]*model.BlockDevice{}, + deviceService: ds, + fileSystemServiceFactory: fssf, + } +} + +func (db *LinuxDeviceBackend) GetBlockDevice(device string) (*model.BlockDevice, error) { + blockDevice, exists := db.blockDevices[device] + if !exists { + return nil, fmt.Errorf("🔴 %s: Could not find block device", device) + } + return blockDevice, nil +} + +// This method is unique in the sense that some file-systems like xfs might require +// that the device be unmounted before any labelling operations can commence. For these +// file systems, we would preprend any label action with an unmount action (if the device is already mounted) +func (db *LinuxDeviceBackend) Label(bd *model.BlockDevice, label string) ([]action.Action, error) { + actions := make([]action.Action, 0) + fss, err := db.fileSystemServiceFactory.Select(bd.FileSystem) + if err != nil { + return nil, err + } + if fss.DoesLabelRequireUnmount() && len(bd.MountPoint) > 0 { + a := db.Umount(bd) + actions = append(actions, a) + } + a := action.NewLabelDeviceAction( + bd.Name, + label, + fss, + ) + return append(actions, a), nil +} + +func (db *LinuxDeviceBackend) Resize(bd *model.BlockDevice) (action.Action, error) { + fss, err := db.fileSystemServiceFactory.Select(bd.FileSystem) + if err != nil { + return nil, err + } + target := bd.Name + if fss.DoesResizeRequireMount() { + if len(bd.MountPoint) == 0 { + return nil, fmt.Errorf("🔴 %s: To resize the %s file system, device must be mounted", fss.GetFileSystem(), bd.Name) + } + target = bd.MountPoint + } + return action.NewResizeDeviceAction( + bd.Name, + target, + fss, + ), nil +} + +func (db *LinuxDeviceBackend) Format(bd *model.BlockDevice, fileSystem model.FileSystem) (action.Action, error) { + fss, err := db.fileSystemServiceFactory.Select(fileSystem) + if err != nil { + return nil, err + } + return action.NewFormatDeviceAction( + bd.Name, + fss, + ), nil +} + +func (db *LinuxDeviceBackend) Mount(bd *model.BlockDevice, target string, options model.MountOptions) action.Action { + return action.NewMountDeviceAction(bd.Name, target, bd.FileSystem, options, db.deviceService) +} + +func (db *LinuxDeviceBackend) Remount(bd *model.BlockDevice, target string, options model.MountOptions) action.Action { + return db.Mount(bd, target, options.Remount()) +} + +func (db *LinuxDeviceBackend) Umount(bd *model.BlockDevice) action.Action { + return action.NewUnmountDeviceAction(bd.Name, bd.MountPoint, db.deviceService) +} + +func (db *LinuxDeviceBackend) From(config *config.Config) error { + // Clear in memory representation of devices + for k := range db.blockDevices { + delete(db.blockDevices, k) + } + + for name := range config.Devices { + d, err := db.deviceService.GetBlockDevice(name) + if err != nil { + return err + } + db.blockDevices[d.Name] = d + } + return nil +} diff --git a/internal/backend/file.go b/internal/backend/file.go new file mode 100644 index 0000000..b41a49e --- /dev/null +++ b/internal/backend/file.go @@ -0,0 +1,105 @@ +package backend + +import ( + "fmt" + "os" + "path" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type FileBackend interface { + CreateDirectory(p string) action.Action + ChangeOwner(p string, uid int, gid int) action.Action + ChangePermissions(p string, perms model.FilePermissions) action.Action + GetDirectory(p string) (*model.File, error) + IsMount(p string) bool + From(config *config.Config) error +} + +type LinuxFileBackend struct { + files map[string]*model.File + fileService service.FileService +} + +func NewLinuxFileBackend(fs service.FileService) *LinuxFileBackend { + return &LinuxFileBackend{ + files: map[string]*model.File{}, + fileService: fs, + } +} + +func (lfb *LinuxFileBackend) CreateDirectory(p string) action.Action { + return action.NewCreateDirectoryAction(p, lfb.fileService) +} + +func (lfb *LinuxFileBackend) ChangeOwner(p string, uid int, gid int) action.Action { + return action.NewChangeOwnerAction(p, uid, gid, lfb.fileService) +} + +func (lfb *LinuxFileBackend) ChangePermissions(p string, perms model.FilePermissions) action.Action { + return action.NewChangePermissions(p, perms, lfb.fileService) +} + +func (lfb *LinuxFileBackend) GetDirectory(p string) (*model.File, error) { + f, exists := lfb.files[p] + if !exists { + return nil, os.ErrNotExist + } + if f.Type != model.Directory { + return nil, fmt.Errorf("🔴 %s: File is not a directory", p) + } + return f, nil +} + +func (lfb *LinuxFileBackend) IsMount(p string) bool { + child, exists := lfb.files[p] + if !exists { + return false + } + if child.Type != model.Directory { + return false + } + parent, exists := lfb.files[path.Dir(p)] + if !exists { + return false + } + if child.DeviceId != parent.DeviceId { + return true + } + return child.InodeNo == parent.InodeNo +} + +func (lfb *LinuxFileBackend) From(config *config.Config) error { + // Clear in memory representation of files + for k := range lfb.files { + delete(lfb.files, k) + } + + for _, cd := range config.Devices { + // For certain file operations, it is essential that we + // can query the parent directory. Therefore, lets pull the + // state of the parent directory + files := []string{cd.MountPoint, path.Dir(cd.MountPoint)} + for _, file := range files { + _, exists := lfb.files[file] + if exists { + continue + } + f, err := lfb.fileService.GetFile(file) + if err != nil { + // Gracefully continue loop if the error is because + // the file does not exist. + if os.IsNotExist(err) { + continue + } + return err + } + lfb.files[f.Path] = f + } + } + return nil +} diff --git a/internal/backend/metrics.go b/internal/backend/metrics.go new file mode 100644 index 0000000..d1fb5b9 --- /dev/null +++ b/internal/backend/metrics.go @@ -0,0 +1,78 @@ +package backend + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type DeviceMetricsBackend interface { + GetBlockDeviceMetrics(name string) (*model.BlockDeviceMetrics, error) + From(config *config.Config) error +} + +type LinuxDeviceMetricsBackend struct { + blockDeviceMetrics map[string]*model.BlockDeviceMetrics + deviceService service.DeviceService + fileSystemServiceFactory service.FileSystemServiceFactory +} + +func NewLinuxDeviceMetricsBackend(ds service.DeviceService, fssf service.FileSystemServiceFactory) *LinuxDeviceMetricsBackend { + return &LinuxDeviceMetricsBackend{ + blockDeviceMetrics: map[string]*model.BlockDeviceMetrics{}, + deviceService: ds, + fileSystemServiceFactory: fssf, + } +} + +func (dmb *LinuxDeviceMetricsBackend) GetBlockDeviceMetrics(name string) (*model.BlockDeviceMetrics, error) { + metrics, exists := dmb.blockDeviceMetrics[name] + if !exists { + return nil, fmt.Errorf("🔴 %s: Could not find block device metrics", name) + } + return metrics, nil +} + +func (dmb *LinuxDeviceMetricsBackend) From(config *config.Config) error { + // Clear in memory representation of metrics and devices + for k := range dmb.blockDeviceMetrics { + delete(dmb.blockDeviceMetrics, k) + } + + for name := range config.Devices { + bd, err := dmb.deviceService.GetBlockDevice(name) + if err != nil { + return err + } + // Can not fetch file system metrics from a device with + // no file system. Therefore, we exit with error if this is the case + // The reason we exit early, rather than continuing is because we want + // to simplify the data view of the Device Resize Backend as much as possible + // With this check, we ensure that all devices in the resize backend have a + // valid file system + if bd.FileSystem == model.Unformatted { + return fmt.Errorf("🔴 %s: Can not resize a device with no file system", bd.Name) + } + fs, err := dmb.fileSystemServiceFactory.Select(bd.FileSystem) + if err != nil { + return err + } + // Block Device Size + bss, err := dmb.deviceService.GetSize(bd.Name) + if err != nil { + return err + } + // File System Size + fss, err := fs.GetSize(bd.Name) + if err != nil { + return err + } + dmb.blockDeviceMetrics[bd.Name] = &model.BlockDeviceMetrics{ + BlockDeviceSize: bss, + FileSystemSize: fss, + } + } + return nil +} diff --git a/internal/backend/owner.go b/internal/backend/owner.go new file mode 100644 index 0000000..a80924b --- /dev/null +++ b/internal/backend/owner.go @@ -0,0 +1,73 @@ +package backend + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type OwnerBackend interface { + GetUser(user string) (*model.User, error) + GetGroup(group string) (*model.Group, error) + From(config *config.Config) error +} + +type LinuxOwnerBackend struct { + users map[string]*model.User + groups map[string]*model.Group + ownerService service.OwnerService +} + +func NewLinuxOwnerBackend(ows service.OwnerService) *LinuxOwnerBackend { + return &LinuxOwnerBackend{ + users: map[string]*model.User{}, + groups: map[string]*model.Group{}, + ownerService: ows, + } +} + +func (lfb *LinuxOwnerBackend) GetUser(user string) (*model.User, error) { + o, exists := lfb.users[user] + if !exists { + return nil, fmt.Errorf("🔴 User %s does not exist", user) + } + return o, nil +} + +func (lfb *LinuxOwnerBackend) GetGroup(group string) (*model.Group, error) { + g, exists := lfb.groups[group] + if !exists { + return nil, fmt.Errorf("🔴 Group %s does not exist", group) + } + return g, nil +} + +func (lfb *LinuxOwnerBackend) From(config *config.Config) error { + // Clear in memory representation of files + for k := range lfb.users { + delete(lfb.users, k) + } + for k := range lfb.groups { + delete(lfb.groups, k) + } + + for _, cd := range config.Devices { + if len(cd.User) > 0 { + o, err := lfb.ownerService.GetUser(cd.User) + if err != nil { + return err + } + lfb.users[cd.User] = o + } + if len(cd.Group) > 0 { + g, err := lfb.ownerService.GetGroup(cd.Group) + if err != nil { + return err + } + lfb.groups[cd.Group] = g + } + } + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index ed37803..2e1f9c7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,99 +1,190 @@ package config import ( - "os" - "fmt" - "flag" "bytes" - "gopkg.in/yaml.v2" - "ebs-bootstrap/internal/service" + "flag" + "fmt" + "os" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "gopkg.in/yaml.v2" +) + +const ( + DefaultMode = model.Healthcheck + DefaultMountOptions = model.MountOptions("defaults") ) -type ConfigDevice struct { - Fs string `yaml:"fs"` - MountPoint string `yaml:"mount_point"` - Owner string `yaml:"owner"` - Group string `yaml:"group"` - Label string `yaml:"label"` - Permissions string `yaml:"permissions"` - Mode string `yaml:"mode"` +type Flag struct { + Config string + Mode string + Remount bool + MountOptions string + ResizeFs bool + ResizeThreshold float64 } -type ConfigGlobal struct { - Mode string `yaml:"mode"` +type Device struct { + Fs model.FileSystem `yaml:"fs"` + MountPoint string `yaml:"mountPoint"` + MountOptions model.MountOptions `yaml:"mountOptions"` + User string `yaml:"user"` + Group string `yaml:"group"` + Label string `yaml:"label"` + Permissions model.FilePermissions `yaml:"permissions"` + Mode model.Mode `yaml:"mode"` + Remount bool `yaml:"remount"` + ResizeFs bool `yaml:"resizeFs"` + ResizeThreshold float64 `yaml:"resizeThreshold"` } +type Defaults struct { + Mode model.Mode `yaml:"mode"` + Remount bool `yaml:"remount"` + MountOptions model.MountOptions `yaml:"mountOptions"` + ResizeFs bool `yaml:"resizeFs"` + ResizeThreshold float64 `yaml:"resizeThreshold"` +} + +type Overrides struct { + Mode model.Mode `yaml:"mode"` + Remount bool `yaml:"remount"` + MountOptions model.MountOptions `yaml:"mountOptions"` + ResizeFs bool `yaml:"resizeFs"` + ResizeThreshold float64 `yaml:"resizeThreshold"` +} + +// We don't export "overrides" as this is an attribute that is used +// internally to store the state of flag overrides type Config struct { - Global ConfigGlobal `yaml:"global"` - Devices map[string]ConfigDevice `yaml:"devices"` + Defaults Defaults `yaml:"defaults"` + Devices map[string]Device `yaml:"devices"` + overrides Overrides } -func New(args []string, dt *service.DeviceTranslator, fs service.FileService) (*Config, error) { - // Generate path of config - cp, err := parseFlags(args[0], args[1:]) +func New(args []string) (*Config, error) { + // Generate config path + f, err := parseFlags(args[0], args[1:]) if err != nil { fmt.Fprint(os.Stderr, err) - return nil, fmt.Errorf("🔴 Failed to parse provided flags") - } - - // Validate the path first - if err := fs.ValidateFile(cp); err != nil { - return nil, err - } + return nil, fmt.Errorf("🔴 Failed to parse provided flags") + } - // Create config structure - config := &Config{} + // Create config structure + c := (&Config{}).setOverrides(f) - // Load config file into memory as byte[] - file, err := os.ReadFile(cp) - if err != nil { + // Load config file into memory + file, err := os.ReadFile(f.Config) + if err != nil { return nil, err - } + } - // Unmarshal YAML file from memory into struct - err = yaml.UnmarshalStrict(file, config) - if err != nil { + // Unmarshal YAML file from memory into struct + err = yaml.UnmarshalStrict(file, c) + if err != nil { fmt.Fprintln(os.Stderr, err) - return nil, fmt.Errorf("🔴 Failed to ingest malformed config") - } - - // Layer modifications to the Config. These modifiers will incrementally - // transform the Config until it reaches a desired state - modifiers := []Modifiers{ - &OwnerModifier{}, - &DeviceModifier{ - DeviceTranslator: dt, - }, - &GroupModifier{}, - &DeviceModeModifier{}, - } - for _, modifier := range modifiers { - err = modifier.Modify(config) - if err != nil { - return nil, err - } - } - return config, nil + return nil, fmt.Errorf("🔴 %s: Failed to ingest malformed config", f.Config) + } + + return c, nil } -func parseFlags(program string, args []string) (string, error) { +func parseFlags(program string, args []string) (*Flag, error) { flags := flag.NewFlagSet(program, flag.ContinueOnError) var buf bytes.Buffer flags.SetOutput(&buf) - // String that contains the configured configuration path - var cp string + // String that contains the configured configuration path + f := &Flag{} // String that contains the mode of bootstrap process - // Set up a CLI flag called "-config" to allow users - // to supply the configuration file - flags.StringVar(&cp, "config", "/etc/ebs-bootstrap/config.yml", "path to config file") + // Set up a CLI flag called "-config" to allow users + // to supply the configuration file + flags.StringVar(&f.Config, "config", "/etc/ebs-bootstrap/config.yml", "path to config file") + flags.StringVar(&f.Mode, "mode", "", "override for mode") + flags.BoolVar(&f.Remount, "remount", false, "override for remount") + flags.StringVar(&f.MountOptions, "mount-options", "", "override for mount options") + flags.BoolVar(&f.ResizeFs, "resize-fs", false, "override for resize filesystem") + flags.Float64Var(&f.ResizeThreshold, "resize-threshold", 0, "override for resize threshold") - // Actually parse the flag - err := flags.Parse(args) + // Actually parse the flag + err := flags.Parse(args) if err != nil { - return "", fmt.Errorf(buf.String()) + return nil, fmt.Errorf(buf.String()) + } + + return f, nil +} + +func (c *Config) setOverrides(f *Flag) *Config { + c.overrides.Mode = model.Mode(f.Mode) + c.overrides.Remount = f.Remount + c.overrides.MountOptions = model.MountOptions(f.MountOptions) + c.overrides.ResizeFs = f.ResizeFs + c.overrides.ResizeThreshold = f.ResizeThreshold + return c +} + +func (c *Config) GetMode(name string) model.Mode { + cd, found := c.Devices[name] + if !found { + return DefaultMode + } + if c.overrides.Mode != model.Empty { + return c.overrides.Mode + } + if cd.Mode != model.Empty { + return cd.Mode + } + if c.Defaults.Mode != model.Empty { + return c.Defaults.Mode + } + return DefaultMode +} + +func (c *Config) GetRemount(name string) bool { + cd, found := c.Devices[name] + if !found { + return false + } + return c.overrides.Remount || c.Defaults.Remount || cd.Remount +} + +func (c *Config) GetMountOptions(name string) model.MountOptions { + cd, found := c.Devices[name] + if !found { + return DefaultMountOptions + } + if len(c.overrides.MountOptions) > 0 { + return c.overrides.MountOptions + } + if len(cd.MountOptions) > 0 { + return cd.MountOptions + } + if len(c.Defaults.MountOptions) > 0 { + return c.Defaults.MountOptions + } + return DefaultMountOptions +} + +func (c *Config) GetResizeFs(name string) bool { + cd, found := c.Devices[name] + if !found { + return false + } + return c.overrides.ResizeFs || c.Defaults.ResizeFs || cd.ResizeFs +} + +func (c *Config) GetResizeThreshold(name string) float64 { + cd, found := c.Devices[name] + if !found { + return 0 + } + if c.overrides.ResizeThreshold > 0 { + return c.overrides.ResizeThreshold + } + if cd.ResizeThreshold > 0 { + return cd.ResizeThreshold } - // Return the configuration path - return cp, nil + return c.Defaults.ResizeThreshold } diff --git a/internal/config/config_test.go b/internal/config/config_test.go deleted file mode 100644 index dc03b3a..0000000 --- a/internal/config/config_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package config - -import ( - "fmt" - "os" - "testing" - "ebs-bootstrap/internal/utils" - "ebs-bootstrap/internal/service" - "github.com/google/go-cmp/cmp" -) - -var dt = &service.DeviceTranslator{ - Table: map[string]string{ - "/dev/xvdf": "/dev/nvme0n1", - "/dev/nvme0n1": "/dev/nvme0n1", - }, -} - -var fs = &service.UnixFileService{} - -func TestConfigParsing(t *testing.T) { - u, g, err := utils.GetCurrentUserGroup() - if err != nil { - t.Error(err) - return - } - subtests := []struct{ - Name string - Data []byte - ExpectedOutput *Config - ExpectedErr error - }{ - { - Name: "Valid Config", - Data: []byte(fmt.Sprintf(`--- -global: - mode: healthcheck -devices: - /dev/xvdf: - fs: "xfs" - mount_point: "/ifmx/dev/root" - owner: "%s" - group: "%s" - permissions: 755 - label: "external-vol"`,u.Name, g.Name)), - ExpectedOutput: &Config{ - Global: ConfigGlobal{ - Mode: "healthcheck", - }, - Devices: map[string]ConfigDevice{ - "/dev/nvme0n1": ConfigDevice{ - Fs: "xfs", - MountPoint: "/ifmx/dev/root", - Owner: u.Uid, - Group: g.Gid, - Permissions: "755", - Label: "external-vol", - Mode: "healthcheck", - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Malformed Config", - Data: []byte(`--- -global: - mode: healthcheck -devices:: - /dev/xvdf: - bad_attribute: false`), - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("🔴 Failed to ingest malformed config"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - configPath, err := createConfigFile(subtest.Data) - if err != nil { - t.Errorf("createConfigFile() [error] %s", err) - } - defer os.Remove(configPath) - - c, err := New( - []string{"ebs-bootstrap-test", "-config", configPath}, - dt, - fs, - ) - if !cmp.Equal(c, subtest.ExpectedOutput) { - t.Errorf("Modify() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, c) - } - utils.CheckError("config.New()", t, subtest.ExpectedErr, err) - }) - } -} - -func TestFlagParsing(t *testing.T) { - u, g, err := utils.GetCurrentUserGroup() - if err != nil { - t.Error(err) - return - } - // Create a variable to the path of a valid config - c, err := createConfigFile([]byte(fmt.Sprintf(`--- -global: - mode: healthcheck -devices: - /dev/xvdf: - fs: "xfs" - mount_point: "/ifmx/dev/root" - owner: "%s" - group: "%s" - permissions: 755 - label: "external-vol"`,u.Uid, g.Gid))) - if err != nil { - t.Errorf("createConfigFile() [error] %s", err) - return - } - - // Create a variable to the current working directory - d, err := os.Getwd() - if err != nil { - t.Errorf("os.Getwd() [error] %s", err) - } - - subtests := []struct{ - Name string - Args []string - ExpectedErr error - }{ - { - Name: "Valid Config", - Args: []string{"ebs-bootstrap-test","-config",c}, - ExpectedErr: nil, - }, - { - Name: "Invalid Config (Directory)", - Args: []string{"ebs-bootstrap-test","-config",d}, - ExpectedErr: fmt.Errorf("🔴 %s is not a regular file", d), - }, - { - Name: "Invalid Config (Non-existent File)", - Args: []string{"ebs-bootstrap-test","-config","/doesnt-exist"}, - ExpectedErr: fmt.Errorf("🔴 /doesnt-exist does not exist"), - }, - { - Name: "Unsupported Flag", - Args: []string{"ebs-bootstrap-test","-unsupported-flag"}, - ExpectedErr: fmt.Errorf("🔴 Failed to parse provided flags"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - _, err := New( - subtest.Args, - dt, - fs, - ) - utils.CheckError("config.New()", t, subtest.ExpectedErr, err) - }) - } -} - -func createConfigFile(data []byte) (string, error) { - f, err := os.CreateTemp("", "config_test_*.yml") - if err != nil { - return "", fmt.Errorf("🔴 Failed to create temporary config file: %v", err) - } - defer f.Close() - if _, err := f.Write(data); err != nil { - return "", fmt.Errorf("🔴 Failed to write to temporary config file: %v", err) - } - return f.Name(), nil -} diff --git a/internal/config/modifers_test.go b/internal/config/modifers_test.go deleted file mode 100644 index c73abb7..0000000 --- a/internal/config/modifers_test.go +++ /dev/null @@ -1,335 +0,0 @@ -package config - -import ( - "testing" - "fmt" - "ebs-bootstrap/internal/utils" - "ebs-bootstrap/internal/service" - "github.com/google/go-cmp/cmp" -) - -func TestOwnerModifier(t *testing.T) { - subtests := []struct{ - Name string - Config *Config - ExpectedOutput *Config - ExpectedErr error - }{ - { - Name: "Existing Owner (Non-Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Owner: "root", - }, - }, - }, - ExpectedOutput: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Owner: "0", - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Existing Owner (Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Owner: "0", - }, - }, - }, - ExpectedOutput: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Owner: "0", - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Non-existent Owner (Non-Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Owner: "non-existent-user", - }, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("user: unknown user non-existent-user"), - }, - { - Name: "Non-existent Owner (Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Owner: "-1", - }, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("user: unknown userid -1"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - om := &OwnerModifier{} - err := om.Modify(subtest.Config) - if subtest.ExpectedOutput != nil { - if !cmp.Equal(subtest.Config, subtest.ExpectedOutput) { - t.Errorf("Modify() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, subtest.Config) - } - } - utils.CheckError("Modify()", t, subtest.ExpectedErr, err) - }) - } -} - -func TestGroupModifier(t *testing.T) { - _, g, err := utils.GetCurrentUserGroup() - if err != nil { - t.Error(err) - return - } - subtests := []struct{ - Name string - Config *Config - ExpectedOutput *Config - ExpectedErr error - }{ - { - Name: "Existing Group (Non-Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Group: g.Name, - }, - }, - }, - ExpectedOutput: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Group: g.Gid, - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Existing Group (Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Group: g.Gid, - }, - }, - }, - ExpectedOutput: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Group: g.Gid, - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Non-existent Group (Non-Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Group: "non-existent-group", - }, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("group: unknown group non-existent-group"), - }, - { - Name: "Non-existent Group (Int)", - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Group: "-1", - }, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("group: unknown groupid -1"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - gm := &GroupModifier{} - err := gm.Modify(subtest.Config) - if subtest.ExpectedOutput != nil { - if !cmp.Equal(subtest.Config, subtest.ExpectedOutput) { - t.Errorf("Modify() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, subtest.Config) - } - } - utils.CheckError("Modify()", t, subtest.ExpectedErr, err) - }) - } -} - -func TestDeviceModifier(t *testing.T) { - subtests := []struct{ - Name string - DeviceTranslator *service.DeviceTranslator - Config *Config - ExpectedOutput *Config - ExpectedErr error - }{ - { - Name: "DeviceTranslator() Hit", - DeviceTranslator: &service.DeviceTranslator{ - Table: map[string]string{ - "/dev/nvme0n1": "/dev/nvme0n1", - "/dev/xvdf": "/dev/nvme0n1", - }, - }, - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvdf": ConfigDevice{}, - }, - }, - ExpectedOutput: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/nvme0n1": ConfigDevice{}, - }, - }, - ExpectedErr: nil, - }, - { - Name: "DeviceTranslator() Miss", - DeviceTranslator: &service.DeviceTranslator{ - Table: map[string]string{}, - }, - Config: &Config{ - Devices: map[string]ConfigDevice{ - "/dev/xvdf": ConfigDevice{}, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("🔴 Could not identify a device with an alias /dev/xvdf"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - dm := &DeviceModifier{DeviceTranslator: subtest.DeviceTranslator} - err := dm.Modify(subtest.Config) - if subtest.ExpectedOutput != nil { - if !cmp.Equal(subtest.Config, subtest.ExpectedOutput) { - t.Errorf("Modify() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, subtest.Config) - } - } - utils.CheckError("Modify()", t, subtest.ExpectedErr, err) - }) - } -} - -func TestDeviceModeModifier(t *testing.T) { - subtests := []struct{ - Name string - Config *Config - ExpectedOutput *Config - ExpectedErr error - }{ - { - Name: "Valid Global Mode, Empty Local Mode", - Config: &Config{ - Global: ConfigGlobal{ - Mode: ValidDeviceModes[0], - }, - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{}, - }, - }, - ExpectedOutput: &Config{ - Global: ConfigGlobal{ - Mode: ValidDeviceModes[0], - }, - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Mode: ValidDeviceModes[0], - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Empty Global Mode, Valid Local Mode", - Config: &Config{ - Global: ConfigGlobal{}, - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Mode: ValidDeviceModes[0], - }, - }, - }, - ExpectedOutput: &Config{ - Global: ConfigGlobal{}, - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Mode: ValidDeviceModes[0], - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Empty Global Mode, Empty Local Mode", - Config: &Config{ - Global: ConfigGlobal{}, - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{}, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("🔴 /dev/xvda: If mode is not provided locally, it must be provided globally"), - }, - { - Name: "Invalid Global Mode, Empty Local Mode", - Config: &Config{ - Global: ConfigGlobal{ - Mode: "non-supported-mode", - }, - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{}, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("🔴 A valid global mode was not provided: Expected=%s Provided=non-supported-mode", ValidDeviceModes), - }, - { - Name: "Empty Global Mode, Invalid Local Mode", - Config: &Config{ - Global: ConfigGlobal{}, - Devices: map[string]ConfigDevice{ - "/dev/xvda": ConfigDevice{ - Mode: "non-supported-mode", - }, - }, - }, - ExpectedOutput: nil, - ExpectedErr: fmt.Errorf("🔴 /dev/xvda: A valid mode was not provided: Expected=%s Provided=non-supported-mode", ValidDeviceModes), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - gm := &DeviceModeModifier{} - err := gm.Modify(subtest.Config) - if subtest.ExpectedOutput != nil { - if !cmp.Equal(subtest.Config, subtest.ExpectedOutput) { - t.Errorf("Modify() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, subtest.Config) - } - } - utils.CheckError("Modify()", t, subtest.ExpectedErr, err) - }) - } -} diff --git a/internal/config/modifier.go b/internal/config/modifier.go new file mode 100644 index 0000000..89ba6b5 --- /dev/null +++ b/internal/config/modifier.go @@ -0,0 +1,63 @@ +package config + +import ( + "log" + "strings" + + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type Modifier interface { + Modify(c *Config) error +} + +type AwsNitroNVMeModifier struct { + nvmeService service.NVMeService + deviceService service.DeviceService +} + +func NewAwsNVMeDriverModifier(nvmeService service.NVMeService, deviceService service.DeviceService) *AwsNitroNVMeModifier { + return &AwsNitroNVMeModifier{ + nvmeService: nvmeService, + deviceService: deviceService, + } +} + +func (andm *AwsNitroNVMeModifier) Modify(c *Config) error { + bds, err := andm.deviceService.GetBlockDevices() + if err != nil { + return err + } + for _, name := range bds { + // Check if device already exists in the config + // No need to make additional queries if this is the case + _, exists := c.Devices[name] + if exists { + continue + } + if !strings.HasPrefix(name, "/dev/nvme") { + continue + } + bdm, err := andm.nvmeService.GetBlockDeviceMapping(name) + if err != nil { + return err + } + cd, exists := c.Devices[bdm] + // We can detect AWS NVMe Devices, but this doesn't neccesarily + // mean they will be managed through configuration + if !exists { + continue + } + log.Printf("🔵 %s: Detected Nitro-based AWS NVMe device => %s", name, bdm) + // Delete the original reference to the device configuration from the + // block device mapping retrieved from the NVMe IoCtl interface and + // replace it with the actual device name + // Before: + // /dev/sdb => *config.Device (a) + // After: + // /dev/nvme0n1 => *config.Device (a) + c.Devices[name] = cd + delete(c.Devices, bdm) + } + return nil +} diff --git a/internal/config/modifiers.go b/internal/config/modifiers.go deleted file mode 100644 index bde2e96..0000000 --- a/internal/config/modifiers.go +++ /dev/null @@ -1,97 +0,0 @@ -package config - -import ( - "fmt" - "os/user" - "strconv" - "slices" - "ebs-bootstrap/internal/service" -) - -var ( - ValidDeviceModes = []string{"healthcheck"} -) - -type Modifiers interface { - Modify(config *Config) (error) -} - -type OwnerModifier struct {} - -func (om *OwnerModifier) Modify(config *Config) (error) { - for key, device := range config.Devices { - var u *user.User; - var err error; - if _, atoiErr := strconv.Atoi(device.Owner); atoiErr != nil { - u, err = user.Lookup(device.Owner) - } else { - u, err = user.LookupId(device.Owner) - } - if err != nil { - return err - } - device.Owner = u.Uid - config.Devices[key] = device - } - return nil -} - -type DeviceModifier struct { - DeviceTranslator *service.DeviceTranslator -} - -func (dm *DeviceModifier) Modify(config *Config) (error) { - for key, device := range config.Devices { - alias, found := dm.DeviceTranslator.Table[key] - if !found { - return fmt.Errorf("🔴 Could not identify a device with an alias %s", key) - } - delete(config.Devices, key) - config.Devices[alias] = device - } - return nil -} - -type GroupModifier struct {} - -func (gm *GroupModifier) Modify(config *Config) (error) { - for key, device := range config.Devices { - var g *user.Group; - var err error; - if _, atoiErr := strconv.Atoi(device.Group); atoiErr != nil { - g, err = user.LookupGroup(device.Group) - } else { - g, err = user.LookupGroupId(device.Group) - } - if err != nil { - return err - } - device.Group = g.Gid - config.Devices[key] = device - } - return nil -} - -type DeviceModeModifier struct {} - -func (dm *DeviceModeModifier) Modify(config *Config) (error) { - if config.Global.Mode != "" && !slices.Contains(ValidDeviceModes, config.Global.Mode) { - return fmt.Errorf("🔴 A valid global mode was not provided: Expected=%s Provided=%s", ValidDeviceModes, config.Global.Mode) - } - - for key, device := range config.Devices { - if device.Mode == "" && config.Global.Mode == "" { - return fmt.Errorf("🔴 %s: If mode is not provided locally, it must be provided globally", key) - } - - if device.Mode != "" && !slices.Contains(ValidDeviceModes, device.Mode) { - return fmt.Errorf("🔴 %s: A valid mode was not provided: Expected=%s Provided=%s", key, ValidDeviceModes, device.Mode) - } - - if device.Mode == "" { - device.Mode = config.Global.Mode - } - config.Devices[key] = device - } - return nil -} diff --git a/internal/config/validator.go b/internal/config/validator.go new file mode 100644 index 0000000..2ff31c2 --- /dev/null +++ b/internal/config/validator.go @@ -0,0 +1,186 @@ +package config + +import ( + "fmt" + "path" + "strings" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/service" +) + +type Validator interface { + Validate(c *Config) error +} + +type DeviceValidator struct { + deviceService service.DeviceService +} + +func NewDeviceValidator(ds service.DeviceService) *DeviceValidator { + return &DeviceValidator{ + deviceService: ds, + } +} + +func (dv *DeviceValidator) Validate(c *Config) error { + for name := range c.Devices { + _, err := dv.deviceService.GetBlockDevice(name) + if err != nil { + return fmt.Errorf("🔴 %s is not a block device", name) + } + } + return nil +} + +type FileSystemValidator struct{} + +func NewFileSystemValidator() *FileSystemValidator { + return &FileSystemValidator{} +} + +func (fsv *FileSystemValidator) Validate(c *Config) error { + for name, device := range c.Devices { + fs, err := model.ParseFileSystem(string(device.Fs)) + if err != nil { + return fmt.Errorf("🔴 %s: '%s' is not a supported file system", name, fs) + } + if fs == model.Unformatted { + return fmt.Errorf("🔴 %s: Must provide a supported file system", name) + } + } + return nil +} + +type ModeValidator struct{} + +func NewModeValidator() *ModeValidator { + return &ModeValidator{} +} + +func (fsv *ModeValidator) Validate(c *Config) error { + mode := string(c.Defaults.Mode) + _, err := model.ParseMode(mode) + if err != nil { + return fmt.Errorf("🔴 '%s' (defaults) is not a global mode", mode) + } + + mode = string(c.overrides.Mode) + _, err = model.ParseMode(mode) + if err != nil { + return fmt.Errorf("🔴 '%s' (-mode) is not a supported mode", mode) + } + + for name, device := range c.Devices { + mode := string(device.Mode) + _, err := model.ParseMode(mode) + if err != nil { + return fmt.Errorf("🔴 %s: %s is not a supported mode", name, mode) + } + } + return nil +} + +type MountPointValidator struct{} + +func NewMountPointValidator() *MountPointValidator { + return &MountPointValidator{} +} + +func (apv *MountPointValidator) Validate(c *Config) error { + for name, device := range c.Devices { + if len(device.MountPoint) == 0 { + continue + } + if !path.IsAbs(device.MountPoint) { + return fmt.Errorf("🔴 %s: %s is not an absolute path", name, device.MountPoint) + } + if device.MountPoint == "/" { + return fmt.Errorf("🔴 %s: Can not be mounted to the root directory", name) + } + } + return nil +} + +type MountOptionsValidator struct{} + +func NewMountOptionsValidator() *MountOptionsValidator { + return &MountOptionsValidator{} +} + +func (mov *MountOptionsValidator) Validate(c *Config) error { + mo := string(c.Defaults.MountOptions) + if err := mov.validate(mo); err != nil { + return fmt.Errorf("🔴 '%s' (defaults) is not a supported mode as %s", mo, err) + } + mo = string(c.overrides.MountOptions) + if err := mov.validate(mo); err != nil { + return fmt.Errorf("🔴 '%s' (-mount-options) is not a supported mode as %s", mo, err) + } + for name, device := range c.Devices { + mo := string(device.MountOptions) + if err := mov.validate(mo); err != nil { + return fmt.Errorf("🔴 %s: '%s' is not a supported mode as %s", name, mo, err) + } + } + return nil +} + +func (mov *MountOptionsValidator) validate(mo string) error { + if strings.Contains(mo, "remount") { + return fmt.Errorf("it prevents unmounted devices from being mounted") + } + if strings.Contains(mo, "bind") { + return fmt.Errorf("bind mounts are not supported for block devices") + } + return nil +} + +type OwnerValidator struct { + ownerService service.OwnerService +} + +func NewOwnerValidator(ows service.OwnerService) *OwnerValidator { + return &OwnerValidator{ + ownerService: ows, + } +} + +func (ov *OwnerValidator) Validate(c *Config) error { + for _, device := range c.Devices { + if len(device.User) > 0 { + _, err := ov.ownerService.GetUser(device.User) + if err != nil { + return err + } + } + if len(device.Group) > 0 { + _, err := ov.ownerService.GetGroup(device.Group) + if err != nil { + return err + } + } + } + return nil +} + +type ResizeThresholdValidator struct{} + +func NewResizeThresholdValidator() *ResizeThresholdValidator { + return &ResizeThresholdValidator{} +} + +func (rtv *ResizeThresholdValidator) Validate(c *Config) error { + if c.Defaults.ResizeThreshold < 0 || c.Defaults.ResizeThreshold > 100 { + return fmt.Errorf("🔴 '%g' (default) must be a floating point between 0 and 100 (inclusive)", c.Defaults.ResizeThreshold) + } + if c.overrides.ResizeThreshold < 0 || c.overrides.ResizeThreshold > 100 { + return fmt.Errorf("🔴 '%g' (-resize-threshold) must be a floating point between 0 and 100 (inclusive)", c.overrides.ResizeThreshold) + } + for name, device := range c.Devices { + if device.ResizeThreshold < 0 || device.ResizeThreshold > 100 { + return fmt.Errorf("🔴 %s: '%g' must be a floating point between 0 and 100 (inclusive)", name, device.ResizeThreshold) + } + } + return nil +} diff --git a/internal/layer/directory.go b/internal/layer/directory.go new file mode 100644 index 0000000..3024c1f --- /dev/null +++ b/internal/layer/directory.go @@ -0,0 +1,68 @@ +package layer + +import ( + "fmt" + "os" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" +) + +type CreateDirectoryLayer struct { + deviceBackend backend.DeviceBackend + fileBackend backend.FileBackend +} + +func NewCreateDirectoryLayer(db backend.DeviceBackend, fb backend.FileBackend) *CreateDirectoryLayer { + return &CreateDirectoryLayer{ + deviceBackend: db, + fileBackend: fb, + } +} + +func (fdl *CreateDirectoryLayer) From(c *config.Config) error { + err := fdl.deviceBackend.From(c) + if err != nil { + return err + } + return fdl.fileBackend.From(c) +} + +func (fdl *CreateDirectoryLayer) Modify(c *config.Config) ([]action.Action, error) { + actions := make([]action.Action, 0) + for name, cd := range c.Devices { + if len(cd.MountPoint) == 0 { + continue + } + d, err := fdl.fileBackend.GetDirectory((cd.MountPoint)) + if err != nil && !os.IsNotExist(err) { + // This layer's responsibility is to create a directory if it doesn't exist. + // Therefore, it won't generate an error if a file doesn't exist at the specified path. + return nil, fmt.Errorf("🔴 %s: %s must be a directory for a device to be mounted to it", name, cd.MountPoint) + } + if d != nil { + continue + } + mode := c.GetMode(name) + a := fdl.fileBackend.CreateDirectory(cd.MountPoint).SetMode(mode) + actions = append(actions, a) + } + return actions, nil +} + +func (fdl *CreateDirectoryLayer) Validate(c *config.Config) error { + for name, cd := range c.Devices { + if len(cd.MountPoint) == 0 { + continue + } + if _, err := fdl.fileBackend.GetDirectory(cd.MountPoint); err != nil { + return fmt.Errorf("🔴 %s: Failed directory validation checks. %s does not exist or is not a directory", name, cd.MountPoint) + } + } + return nil +} + +func (fdl *CreateDirectoryLayer) Warning() string { + return DisabledWarning +} diff --git a/internal/layer/format.go b/internal/layer/format.go new file mode 100644 index 0000000..3d29643 --- /dev/null +++ b/internal/layer/format.go @@ -0,0 +1,68 @@ +package layer + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" +) + +type FormatDeviceLayer struct { + deviceBackend backend.DeviceBackend +} + +func NewFormatDeviceLayer(db backend.DeviceBackend) *FormatDeviceLayer { + return &FormatDeviceLayer{ + deviceBackend: db, + } +} + +func (fdl *FormatDeviceLayer) From(c *config.Config) error { + return fdl.deviceBackend.From(c) +} + +func (fdl *FormatDeviceLayer) Modify(c *config.Config) ([]action.Action, error) { + actions := make([]action.Action, 0) + for name, cd := range c.Devices { + bd, err := fdl.deviceBackend.GetBlockDevice(name) + if err != nil { + return nil, err + } + if bd.FileSystem == cd.Fs { + continue + } + if cd.Fs == model.Unformatted { + return nil, fmt.Errorf("🔴 %s: Can not erase the file system of a device", bd.Name) + } + if bd.FileSystem != model.Unformatted { + return nil, fmt.Errorf("🔴 %s: Can not format a device that already has a %s file system", bd.Name, bd.FileSystem) + } + a, err := fdl.deviceBackend.Format(bd, cd.Fs) + if err != nil { + return nil, err + } + mode := c.GetMode(name) + a = a.SetMode(mode) + actions = append(actions, a) + } + return actions, nil +} + +func (fdl *FormatDeviceLayer) Validate(c *config.Config) error { + for name, cd := range c.Devices { + d, err := fdl.deviceBackend.GetBlockDevice(name) + if err != nil { + return err + } + if d.FileSystem != cd.Fs { + return fmt.Errorf("🔴 %s: Failed file system validation checks. Expected=%s, Actual=%s", name, cd.Fs, d.FileSystem) + } + } + return nil +} + +func (fdl *FormatDeviceLayer) Warning() string { + return "Formatting larger disks can take several seconds ⌛" +} diff --git a/internal/layer/label.go b/internal/layer/label.go new file mode 100644 index 0000000..bc4453d --- /dev/null +++ b/internal/layer/label.go @@ -0,0 +1,74 @@ +package layer + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" +) + +type LabelDeviceLayer struct { + deviceBackend backend.DeviceBackend +} + +func NewLabelDeviceLayer(db backend.DeviceBackend) *LabelDeviceLayer { + return &LabelDeviceLayer{ + deviceBackend: db, + } +} + +func (fdl *LabelDeviceLayer) From(c *config.Config) error { + return fdl.deviceBackend.From(c) +} + +func (fdl *LabelDeviceLayer) Modify(c *config.Config) ([]action.Action, error) { + actions := make([]action.Action, 0) + for name, cd := range c.Devices { + bd, err := fdl.deviceBackend.GetBlockDevice(name) + if err != nil { + return nil, err + } + if len(cd.Label) == 0 { + continue + } + if bd.Label == cd.Label { + continue + } + if bd.FileSystem == model.Unformatted { + return nil, fmt.Errorf("🔴 %s: Can not label a device with no file system", bd.Name) + } + mode := c.GetMode(name) + // Labelling a device can potentially require unmounting it first + // Therefore, multiple actions may be returned: Label Actions (las) + las, err := fdl.deviceBackend.Label(bd, cd.Label) + if err != nil { + return nil, err + } + for _, la := range las { + actions = append(actions, la.SetMode(mode)) + } + } + return actions, nil +} + +func (fdl *LabelDeviceLayer) Validate(c *config.Config) error { + for name, cd := range c.Devices { + if len(cd.Label) == 0 { + continue + } + d, err := fdl.deviceBackend.GetBlockDevice(name) + if err != nil { + return err + } + if d.Label != cd.Label { + return fmt.Errorf("🔴 %s: Failed label validation checks. Expected=%s, Actual=%s", name, cd.Label, d.Label) + } + } + return nil +} + +func (fdl *LabelDeviceLayer) Warning() string { + return "Certain file systems require that devices be unmounted prior to labeling" +} diff --git a/internal/layer/layer.go b/internal/layer/layer.go new file mode 100644 index 0000000..dc38577 --- /dev/null +++ b/internal/layer/layer.go @@ -0,0 +1,100 @@ +package layer + +import ( + "log" + "math" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/config" +) + +const ( + InitialInterval = 200 * time.Millisecond + Multiplier = 2 + MaxRetries = 3 +) + +const ( + DisabledWarning = "" +) + +type Layer interface { + From(config *config.Config) error + Modify(c *config.Config) ([]action.Action, error) + Validate(config *config.Config) error + Warning() string +} + +type LayerExecutor interface { + Execute(layers []Layer) error +} + +type ExponentialBackoffLayerExecutor struct { + backoff backoff.BackOff + actionExecutor action.ActionExecutor + config *config.Config +} + +func NewExponentialBackoffLayerExecutor(c *config.Config, ae action.ActionExecutor) *ExponentialBackoffLayerExecutor { + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = InitialInterval + // Disable randomisation of the calculation of the next desired backoff duration + bo.RandomizationFactor = 0 + // Set the multiplier for the exponential backoff, using the square root of the provided Multiplier. + bo.Multiplier = math.Sqrt(Multiplier) + // Calculate the maximum elapsed time for the backoff strategy based on the provided MaxRetries and InitialInterval. + // This formula calculates the maximum elapsed time as a geometric series sum for a given number of retries and interval. + bo.MaxElapsedTime = time.Duration((math.Pow(Multiplier, MaxRetries)-1)/(Multiplier-1)) * InitialInterval + bo.MaxInterval = InitialInterval * time.Duration(math.Pow(Multiplier, MaxRetries-1)) + return &ExponentialBackoffLayerExecutor{ + backoff: bo, + actionExecutor: ae, + config: c, + } +} + +func (le *ExponentialBackoffLayerExecutor) Execute(layers []Layer) error { + for _, layer := range layers { + err := layer.From(le.config) + if err != nil { + return err + } + actions, err := layer.Modify(le.config) + if err != nil { + return err + } + // Only print warning if actions are detected and a valid warning + // message is provided + if warning := layer.Warning(); len(actions) > 0 && warning != DisabledWarning { + log.Printf("🟠 %s", warning) + } + err = le.actionExecutor.Execute(actions) + if err != nil { + return err + } + // Reset exponential backoff timer + le.backoff.Reset() + err = backoff.Retry(func() error { + return le.validate(layer) + }, le.backoff) + if err != nil { + return err + } + } + log.Println("🟢 Passed all validation checks") + return nil +} + +func (le *ExponentialBackoffLayerExecutor) validate(layer Layer) error { + // Any potential errors that arise from ingesting the configuration + // are most likely persistent. Therefore, we wrap any errors produced + // from layer.From() as a backoff.Permanent so that it can bypass the + // exponential backoff algorithm + err := layer.From(le.config) + if err != nil { + return backoff.Permanent(err) + } + return layer.Validate(le.config) +} diff --git a/internal/layer/mount.go b/internal/layer/mount.go new file mode 100644 index 0000000..3cf5b23 --- /dev/null +++ b/internal/layer/mount.go @@ -0,0 +1,93 @@ +package layer + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" + "github.com/reecetech/ebs-bootstrap/internal/model" +) + +type MountDeviceLayer struct { + deviceBackend backend.DeviceBackend + fileBackend backend.FileBackend +} + +func NewMountDeviceLayer(db backend.DeviceBackend, fb backend.FileBackend) *MountDeviceLayer { + return &MountDeviceLayer{ + deviceBackend: db, + fileBackend: fb, + } +} + +func (fdl *MountDeviceLayer) From(c *config.Config) error { + err := fdl.deviceBackend.From(c) + if err != nil { + return err + } + return fdl.fileBackend.From(c) +} + +func (fdl *MountDeviceLayer) Modify(c *config.Config) ([]action.Action, error) { + actions := make([]action.Action, 0) + for name, cd := range c.Devices { + bd, err := fdl.deviceBackend.GetBlockDevice(name) + if err != nil { + return nil, err + } + if len(cd.MountPoint) == 0 { + continue + } + if bd.FileSystem == model.Unformatted { + return nil, fmt.Errorf("🔴 %s: Can not mount a device with no file system", bd.Name) + } + mode := c.GetMode(name) + mo := c.GetMountOptions(name) + if bd.MountPoint == cd.MountPoint { + if c.GetRemount(name) { + a := fdl.deviceBackend.Remount(bd, cd.MountPoint, mo).SetMode(mode) + actions = append(actions, a) + } + } else { + _, err = fdl.fileBackend.GetDirectory(cd.MountPoint) + if err != nil { + // We prevent users from mounting devices to symbolic links as this would cause + // validation checks to consistently fail. We could evaluate symbolic links + // prior, but this just adds complexity we do not need right now + return nil, fmt.Errorf("🔴 %s: %s must exist as a directory before it can be mounted", name, cd.MountPoint) + } + if fdl.fileBackend.IsMount(cd.MountPoint) { + return nil, fmt.Errorf("🔴 %s: %s is already mounted by another device", name, cd.MountPoint) + } + // If mount point already exists, then lets unmount it first + if len(bd.MountPoint) > 0 { + a := fdl.deviceBackend.Umount(bd).SetMode(mode) + actions = append(actions, a) + } + a := fdl.deviceBackend.Mount(bd, cd.MountPoint, mo).SetMode(mode) + actions = append(actions, a) + } + } + return actions, nil +} + +func (fdl *MountDeviceLayer) Validate(c *config.Config) error { + for name, cd := range c.Devices { + if len(cd.MountPoint) == 0 { + continue + } + bd, err := fdl.deviceBackend.GetBlockDevice(name) + if err != nil { + return err + } + if bd.MountPoint != cd.MountPoint { + return fmt.Errorf("🔴 %s: Failed mountpoint validation checks. Device not mounted to %s", name, bd.MountPoint) + } + } + return nil +} + +func (fdl *MountDeviceLayer) Warning() string { + return "Devices mounted to a location, not specified in the configuration, will be unmounted" +} diff --git a/internal/layer/owner.go b/internal/layer/owner.go new file mode 100644 index 0000000..b39ddad --- /dev/null +++ b/internal/layer/owner.go @@ -0,0 +1,124 @@ +package layer + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" +) + +type ChangeOwnerLayer struct { + ownerBackend backend.OwnerBackend + fileBackend backend.FileBackend +} + +func NewChangeOwnerLayer(ub backend.OwnerBackend, fb backend.FileBackend) *ChangeOwnerLayer { + return &ChangeOwnerLayer{ + ownerBackend: ub, + fileBackend: fb, + } +} + +func (fdl *ChangeOwnerLayer) From(c *config.Config) error { + err := fdl.ownerBackend.From(c) + if err != nil { + return err + } + return fdl.fileBackend.From(c) +} + +func (fdl *ChangeOwnerLayer) Modify(c *config.Config) ([]action.Action, error) { + actions := make([]action.Action, 0) + for name, cd := range c.Devices { + if len(cd.MountPoint) == 0 { + continue + } + if len(cd.User) == 0 && len(cd.Group) == 0 { + continue + } + + d, err := fdl.fileBackend.GetDirectory(cd.MountPoint) + if err != nil { + return nil, fmt.Errorf("🔴 %s is either not a directory or does not exist", cd.MountPoint) + } + + // Default the values of User ID and Group ID to that + // of the directory + uid := d.Uid + gid := d.Gid + + if len(cd.User) > 0 { + u, err := fdl.ownerBackend.GetUser(cd.User) + if err != nil { + return nil, err + } + uid = u.Uid + } + + if len(cd.Group) > 0 { + g, err := fdl.ownerBackend.GetGroup(cd.Group) + if err != nil { + return nil, err + } + gid = g.Gid + } + + if d.Uid == uid && d.Gid == gid { + continue + } + mode := c.GetMode(name) + a := fdl.fileBackend.ChangeOwner(cd.MountPoint, uid, gid).SetMode(mode) + actions = append(actions, a) + } + return actions, nil +} + +func (fdl *ChangeOwnerLayer) Validate(c *config.Config) error { + for name, cd := range c.Devices { + if len(cd.MountPoint) == 0 { + continue + } + if len(cd.User) == 0 && len(cd.Group) == 0 { + continue + } + + d, err := fdl.fileBackend.GetDirectory(cd.MountPoint) + if err != nil { + return fmt.Errorf("🔴 %s: Failed ownership validation checks. %s is either not a directory or does not exist", name, cd.MountPoint) + } + + // Default the values of User ID and Group ID to that + // of the directory + uid := d.Uid + gid := d.Gid + + if len(cd.User) > 0 { + u, err := fdl.ownerBackend.GetUser(cd.User) + if err != nil { + return err + } + uid = u.Uid + } + + if len(cd.Group) > 0 { + g, err := fdl.ownerBackend.GetGroup(cd.Group) + if err != nil { + return err + } + gid = g.Gid + } + + if d.Uid != uid { + return fmt.Errorf("🔴 %s: Failed ownership validation checks. %s User Expected=%d, Actual=%d", name, cd.MountPoint, d.Uid, uid) + } + if d.Gid != gid { + return fmt.Errorf("🔴 %s: Failed ownership validation checks. %s Group Expected=%d, Actual=%d", name, cd.MountPoint, d.Gid, gid) + } + } + return nil +} + +func (fdl *ChangeOwnerLayer) Warning() string { + return DisabledWarning +} diff --git a/internal/layer/permissions.go b/internal/layer/permissions.go new file mode 100644 index 0000000..99a1e7d --- /dev/null +++ b/internal/layer/permissions.go @@ -0,0 +1,74 @@ +package layer + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" +) + +type ChangePermissionsLayer struct { + fileBackend backend.FileBackend +} + +func NewChangePermissionsLayer(fb backend.FileBackend) *ChangePermissionsLayer { + return &ChangePermissionsLayer{ + fileBackend: fb, + } +} + +func (fdl *ChangePermissionsLayer) From(c *config.Config) error { + return fdl.fileBackend.From(c) +} + +func (fdl *ChangePermissionsLayer) Modify(c *config.Config) ([]action.Action, error) { + actions := make([]action.Action, 0) + for name, cd := range c.Devices { + if len(cd.MountPoint) == 0 { + continue + } + if cd.Permissions == 0 { + continue + } + + d, err := fdl.fileBackend.GetDirectory(cd.MountPoint) + if err != nil { + return nil, fmt.Errorf("🔴 %s is either not a directory or does not exist", cd.MountPoint) + } + + if d.Permissions == cd.Permissions { + continue + } + mode := c.GetMode(name) + a := fdl.fileBackend.ChangePermissions(cd.MountPoint, cd.Permissions).SetMode(mode) + actions = append(actions, a) + } + return actions, nil +} + +func (fdl *ChangePermissionsLayer) Validate(c *config.Config) error { + for name, cd := range c.Devices { + if len(cd.MountPoint) == 0 { + continue + } + // When permissions is not provided + if cd.Permissions == 0 { + continue + } + + d, err := fdl.fileBackend.GetDirectory(cd.MountPoint) + if err != nil { + return fmt.Errorf("🔴 %s: Failed ownership validation checks. %s is either not a directory or does not exist", name, cd.MountPoint) + } + + if d.Permissions != cd.Permissions { + return fmt.Errorf("🔴 %s: Failed permissions validation checks. %s Permissions Expected=%#o, Actual=%#o", name, cd.MountPoint, cd.Permissions, d.Permissions) + } + } + return nil +} + +func (fdl *ChangePermissionsLayer) Warning() string { + return DisabledWarning +} diff --git a/internal/layer/resize.go b/internal/layer/resize.go new file mode 100644 index 0000000..34a947c --- /dev/null +++ b/internal/layer/resize.go @@ -0,0 +1,83 @@ +package layer + +import ( + "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/action" + "github.com/reecetech/ebs-bootstrap/internal/backend" + "github.com/reecetech/ebs-bootstrap/internal/config" +) + +type ResizeDeviceLayer struct { + deviceBackend backend.DeviceBackend + deviceMetricsBackend backend.DeviceMetricsBackend +} + +func NewResizeDeviceLayer(db backend.DeviceBackend, dmb backend.DeviceMetricsBackend) *ResizeDeviceLayer { + return &ResizeDeviceLayer{ + deviceBackend: db, + deviceMetricsBackend: dmb, + } +} + +func (fdl *ResizeDeviceLayer) From(c *config.Config) error { + err := fdl.deviceBackend.From(c) + if err != nil { + return err + } + return fdl.deviceMetricsBackend.From(c) +} + +func (fdl *ResizeDeviceLayer) Modify(c *config.Config) ([]action.Action, error) { + actions := make([]action.Action, 0) + for name := range c.Devices { + if !c.GetResizeFs(name) { + continue + } + bd, err := fdl.deviceBackend.GetBlockDevice(name) + if err != nil { + return nil, err + } + metrics, err := fdl.deviceMetricsBackend.GetBlockDeviceMetrics(name) + if err != nil { + return nil, err + } + mode := c.GetMode(name) + rt := c.GetResizeThreshold(name) + // If the resize threshold is set to 0, always attempt to resize the block device. + // For the currently supported file systems xfs and ext4, the commands to + // resize the block device are idempotent + if rt == 0 || metrics.ShouldResize(rt) { + a, err := fdl.deviceBackend.Resize(bd) + if err != nil { + return nil, err + } + a = a.SetMode(mode) + actions = append(actions, a) + } + } + return actions, nil +} + +func (fdl *ResizeDeviceLayer) Validate(c *config.Config) error { + for name := range c.Devices { + if !c.GetResizeFs(name) { + continue + } + metrics, err := fdl.deviceMetricsBackend.GetBlockDeviceMetrics(name) + if err != nil { + return err + } + rt := c.GetResizeThreshold(name) + // If the resize threshold is 0, then the device would always be resized + // Therefore, lets ignore that case from our validation checks + if rt > 0 && metrics.ShouldResize(rt) { + return fmt.Errorf("🔴 %s: Failed to resize file system. File System=%d Block Device=%d (bytes)", name, metrics.FileSystemSize, metrics.BlockDeviceSize) + } + } + return nil +} + +func (fdl *ResizeDeviceLayer) Warning() string { + return DisabledWarning +} diff --git a/internal/model/action.go b/internal/model/action.go new file mode 100644 index 0000000..d8c923a --- /dev/null +++ b/internal/model/action.go @@ -0,0 +1,22 @@ +package model + +import "fmt" + +type Mode string + +const ( + Empty Mode = "" + Healthcheck Mode = "healthcheck" + Prompt Mode = "prompt" + Force Mode = "force" +) + +func ParseMode(s string) (Mode, error) { + m := Mode(s) + switch m { + case Empty, Healthcheck, Prompt, Force: + return m, nil + default: + return m, fmt.Errorf("🔴 %s: Mode is not supported", s) + } +} diff --git a/internal/model/device.go b/internal/model/device.go new file mode 100644 index 0000000..36b0816 --- /dev/null +++ b/internal/model/device.go @@ -0,0 +1,38 @@ +package model + +import ( + "slices" + "strings" +) + +type BlockDevice struct { + Name string + Uuid string + MountPoint string + FileSystem FileSystem + Label string +} + +type MountOptions string + +func (mop MountOptions) Remount() MountOptions { + mops := strings.Split(string(mop), ",") + // Return index of remount option (if it exists) + // Return -1 if not found + index := slices.Index(mops, "remount") + if index < 0 { + mops = append(mops, "remount") + } + return MountOptions(strings.Join(mops, ",")) +} + +type BlockDeviceMetrics struct { + FileSystemSize uint64 + BlockDeviceSize uint64 +} + +func (bdm *BlockDeviceMetrics) ShouldResize(threshold float64) bool { + // Minimum File System Size (mfss) + mfss := float64(bdm.BlockDeviceSize) * (threshold / 100) + return float64(bdm.FileSystemSize) < mfss +} diff --git a/internal/model/file.go b/internal/model/file.go new file mode 100644 index 0000000..8ec1255 --- /dev/null +++ b/internal/model/file.go @@ -0,0 +1,60 @@ +package model + +import ( + "fmt" + "io/fs" + "strconv" +) + +type FileType int + +const ( + RegularFile FileType = 0 + Directory FileType = 1 + Special FileType = 2 +) + +type File struct { + Path string + Type FileType + DeviceId int + InodeNo int + Uid int // User ID + Gid int // Group ID + Permissions FilePermissions +} + +type FilePermissions uint32 + +// It is useful to be able to convert FilePermissions back into the fs.FileMode +// type which is expected by Go standard libraries +func (p FilePermissions) Perm() fs.FileMode { + return fs.FileMode(p) +} + +// Linux File Permission bits are typically represented as octals: e.g 0755. +// Some users may feel comfortable representing file permission bits as decimals: +// e.g 755. While the latter is not considered an octal, lets not punish them +// for a behaviour that has been ingrained by tools like chmod. +// `strconv.ParseUint` has the ability to force the intepreation of a string as a base-8 +// unsigned integer +func (p *FilePermissions) UnmarshalYAML(unmarshal func(interface{}) error) error { + var ps string + if err := unmarshal(&ps); err != nil { + return err + } + if len(ps) == 0 { + *p = FilePermissions(0) + return nil + } + // Base: 8, Bit Length: 32 + mode, err := strconv.ParseUint(ps, 8, 32) + if err != nil { + return fmt.Errorf("🔴 invalid permission value. '%v' must be a valid octal number", ps) + } + if mode > 0777 { + return fmt.Errorf("🔴 invalid permission value. '%#o' exceeds the maximum allowed value (0777)", mode) + } + *p = FilePermissions(mode) + return nil +} diff --git a/internal/model/filesystem.go b/internal/model/filesystem.go new file mode 100644 index 0000000..01b8f21 --- /dev/null +++ b/internal/model/filesystem.go @@ -0,0 +1,21 @@ +package model + +import "fmt" + +type FileSystem string + +const ( + Unformatted FileSystem = "" + Ext4 FileSystem = "ext4" + Xfs FileSystem = "xfs" +) + +func ParseFileSystem(s string) (FileSystem, error) { + fst := FileSystem(s) + switch fst { + case Unformatted, Ext4, Xfs: + return fst, nil + default: + return fst, fmt.Errorf("🔴 %s: File system is not supported", s) + } +} diff --git a/internal/model/owner.go b/internal/model/owner.go new file mode 100644 index 0000000..e6bd3a2 --- /dev/null +++ b/internal/model/owner.go @@ -0,0 +1,11 @@ +package model + +type User struct { + Name string + Uid int +} + +type Group struct { + Name string + Gid int +} diff --git a/internal/service/device.go b/internal/service/device.go index 9908772..8523f8d 100644 --- a/internal/service/device.go +++ b/internal/service/device.go @@ -3,43 +3,62 @@ package service import ( "encoding/json" "fmt" - "strings" - "ebs-bootstrap/internal/utils" + "strconv" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/utils" ) // Device Service Interface [START] -type DeviceInfo struct { - Name string - Label string - Fs string - MountPoint string -} - type DeviceService interface { + GetSize(name string) (uint64, error) // bytes GetBlockDevices() ([]string, error) - GetDeviceInfo(device string) (*DeviceInfo, error) + GetBlockDevice(name string) (*model.BlockDevice, error) + Mount(source string, target string, fs model.FileSystem, options model.MountOptions) error + Umount(source string, target string) error } // Device Service Interface [END] type LinuxDeviceService struct { - Runner utils.Runner + RunnerFactory utils.RunnerFactory } type LsblkBlockDeviceResponse struct { - BlockDevices []LsblkBlockDevice `json:"blockdevices"` + BlockDevices []LsblkBlockDevice `json:"blockdevices"` } type LsblkBlockDevice struct { - Name string `json:"name"` - Label string `json:"label"` - FsType string `json:"fstype"` - MountPoint string `json:"mountpoint"` + Name *string `json:"name"` + Uuid *string `json:"uuid"` + Label *string `json:"label"` + FsType *string `json:"fstype"` + MountPoint *string `json:"mountpoint"` +} + +func NewLinuxDeviceService(rc utils.RunnerFactory) *LinuxDeviceService { + return &LinuxDeviceService{ + RunnerFactory: rc, + } +} + +func (du *LinuxDeviceService) GetSize(name string) (uint64, error) { + r := du.RunnerFactory.Select(utils.BlockDev) + output, err := r.Command("--getsize64", name) + if err != nil { + return 0, err + } + b, err := strconv.ParseUint(output, 10, 64) + if err != nil { + return 0, fmt.Errorf("🔴 Failed to cast block device size to unsigned 64-bit integer") + } + return b, nil } func (du *LinuxDeviceService) GetBlockDevices() ([]string, error) { - output, err := du.Runner.Command("lsblk", "--nodeps", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT", "-J") + r := du.RunnerFactory.Select(utils.Lsblk) + output, err := r.Command("--nodeps", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT", "-J") if err != nil { return nil, err } @@ -48,68 +67,54 @@ func (du *LinuxDeviceService) GetBlockDevices() ([]string, error) { if err != nil { return nil, err } - d := make([]string,len(lbd.BlockDevices)) - for i, _ := range d { - d[i] = "/dev/" + lbd.BlockDevices[i].Name + d := make([]string, len(lbd.BlockDevices)) + for i := range d { + d[i] = "/dev/" + utils.SafeString(lbd.BlockDevices[i].Name) } return d, nil } -func (du *LinuxDeviceService) GetDeviceInfo(device string) (*DeviceInfo, error) { - output, err := du.Runner.Command("lsblk", "--nodeps", "-o", "NAME,LABEL,FSTYPE,MOUNTPOINT", "-J", device) +func (du *LinuxDeviceService) GetBlockDevice(name string) (*model.BlockDevice, error) { + r := du.RunnerFactory.Select(utils.Lsblk) + output, err := r.Command("--nodeps", "-o", "NAME,UUID,LABEL,FSTYPE,MOUNTPOINT", "-J", name) if err != nil { return nil, err } - bd := &LsblkBlockDeviceResponse{} - err = json.Unmarshal([]byte(output), bd) + lbd := &LsblkBlockDeviceResponse{} + err = json.Unmarshal([]byte(output), lbd) if err != nil { return nil, err } - if len(bd.BlockDevices) != 1 { - return nil, fmt.Errorf("🔴 [%s] An unexpected number of block devices were returned: Expected=1 Actual=%d", device, len(bd.BlockDevices)) + if len(lbd.BlockDevices) != 1 { + return nil, fmt.Errorf("🔴 %s: An unexpected number of block devices were returned: Expected=1 Actual=%d", name, len(lbd.BlockDevices)) } - return &DeviceInfo{ - Name: "/dev/" + bd.BlockDevices[0].Name, - Label: bd.BlockDevices[0].Label, - Fs: bd.BlockDevices[0].FsType, - MountPoint: bd.BlockDevices[0].MountPoint, + fst, err := model.ParseFileSystem(utils.SafeString(lbd.BlockDevices[0].FsType)) + if err != nil { + return nil, err + } + return &model.BlockDevice{ + Name: "/dev/" + utils.SafeString(lbd.BlockDevices[0].Name), + Uuid: utils.SafeString(lbd.BlockDevices[0].Uuid), + Label: utils.SafeString(lbd.BlockDevices[0].Label), + FileSystem: fst, + MountPoint: utils.SafeString(lbd.BlockDevices[0].MountPoint), }, nil } -// Device Translator Service Interface [START] - -type DeviceTranslator struct { - Table map[string]string -} - -type DeviceTranslatorService interface { - GetTranslator() *DeviceTranslator -} - -type EbsDeviceTranslator struct { - DeviceService DeviceService - NVMeService NVMeService +func (du *LinuxDeviceService) Mount(source string, target string, fs model.FileSystem, options model.MountOptions) error { + r := du.RunnerFactory.Select(utils.Mount) + _, err := r.Command(source, "-t", string(fs), "-o", string(options), target) + if err != nil { + return err + } + return nil } -// Device Translator Service Interface [END] - -func (edt *EbsDeviceTranslator) GetTranslator() (*DeviceTranslator, error) { - dt := &DeviceTranslator{} - dt.Table = make(map[string]string) - devices, err := edt.DeviceService.GetBlockDevices() +func (du *LinuxDeviceService) Umount(source string, target string) error { + r := du.RunnerFactory.Select(utils.Umount) + _, err := r.Command(target) if err != nil { - return nil, err - } - for _, device := range(devices) { - alias := device - if strings.HasPrefix(device, "/dev/nvme") { - alias, err = edt.NVMeService.GetBlockDeviceMapping(device) - if err != nil { - return nil, err - } - } - dt.Table[alias] = device - dt.Table[device] = alias + return err } - return dt, nil + return nil } diff --git a/internal/service/device_test.go b/internal/service/device_test.go deleted file mode 100644 index e84cebe..0000000 --- a/internal/service/device_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package service - -import ( - "testing" - "fmt" - "ebs-bootstrap/internal/utils" - "github.com/google/go-cmp/cmp" -) - -type deviceMockRunner struct { - Output string - Error error -} - -func (mr *deviceMockRunner) Command(name string, arg ...string) (string, error) { - return mr.Output, mr.Error -} - -func TestGetBlockDevices(t *testing.T) { - mr := &deviceMockRunner{ - Output: `{"blockdevices": [ - {"name":"nvme1n1", "label":"external-vol", "fstype":"xfs", "mountpoint":"/ifmx/dev/root"}, - {"name":"nvme0n1", "label":null, "fstype":null, "mountpoint":null} - ]}`, - Error: nil, - } - expectedOutput := []string{"/dev/nvme1n1", "/dev/nvme0n1"} - - t.Run("Get Block Devices", func(t *testing.T) { - du := &LinuxDeviceService{mr} - d, err := du.GetBlockDevices() - if !cmp.Equal(d, expectedOutput) { - t.Errorf("GetBlockDevices() [output] mismatch: Expected=%v Actual=%v", expectedOutput, d) - } - utils.CheckError("GetBlockDevices()", t, nil, err) - }) -} - -func TestGetDeviceInfo(t *testing.T) { - deviceNotFoundErr := fmt.Errorf("🔴 /dev/nvme0n1 not a block device") - - subtests := []struct { - Name string - Device string - MockRunner *deviceMockRunner - ExpectedOutput *DeviceInfo - ExpectedErr error - }{ - { - Name: "Get Device Info for /dev/nvme0n1", - Device: "/dev/nvme0n1", - MockRunner: &deviceMockRunner{ - Output: `{"blockdevices":[{"name":"nvme0n1","label":"external-vol","fstype":"xfs","mountpoint":"/mnt/app"}]}`, - Error: nil, - }, - ExpectedOutput: &DeviceInfo{ - Name: "/dev/nvme0n1", - Label: "external-vol", - Fs: "xfs", - MountPoint: "/mnt/app", - }, - ExpectedErr: nil, - }, - { - Name: "Get Device Info for /dev/nvme0n1 (No Fs,Label,Mountpoint)", - Device: "/dev/nvme0n1", - MockRunner: &deviceMockRunner{ - Output: `{"blockdevices":[{"name":"nvme0n1","label":null,"fstype":null,"mountpoint":null}]}`, - Error: nil, - }, - ExpectedOutput: &DeviceInfo{ - Name: "/dev/nvme0n1", - Label: "", - Fs: "", - MountPoint: "", - }, - ExpectedErr: nil, - }, - { - Name: "Get Device Info for Missing Device", - Device: "/dev/nvme0n1", - MockRunner: &deviceMockRunner{ - Output: "", - Error: deviceNotFoundErr, - }, - ExpectedOutput: nil, - ExpectedErr: deviceNotFoundErr, - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - du := &LinuxDeviceService{subtest.MockRunner} - di, err := du.GetDeviceInfo(subtest.Device) - if !cmp.Equal(di, subtest.ExpectedOutput) { - t.Errorf("GetDeviceInfo() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, di) - } - utils.CheckError("GetDeviceInfo()", t, subtest.ExpectedErr, err) - }) - } -} - -type mockDeviceService struct { - getBlockDevices func() ([]string, error) -} - -func (ds *mockDeviceService) GetBlockDevices() ([]string, error) { - return ds.getBlockDevices() -} - -func (ds *mockDeviceService) GetDeviceInfo(device string) (*DeviceInfo, error) { - return nil, fmt.Errorf("🔴 GetDeviceInfo() not implemented") -} - -type mockNVMeService struct { - getBlockDeviceMapping func(device string) (string, error) -} - -func (ns *mockNVMeService) GetBlockDeviceMapping(device string) (string, error) { - return ns.getBlockDeviceMapping(device) -} - -func TestEbsDeviceTranslator(t *testing.T) { - subtests := []struct{ - Name string - DeviceService DeviceService - NVMeService NVMeService - ExpectedOutput *DeviceTranslator - ExpectedErr error - }{ - { - Name: "Get DeviceTranslator for EBS NVME Device", - DeviceService: &mockDeviceService{ - getBlockDevices: func() ([]string, error) { - return []string{"/dev/nvme0n1"}, nil - }, - }, - NVMeService: &mockNVMeService { - getBlockDeviceMapping: func(device string) (string, error) { - return "/dev/xvdf", nil - }, - }, - ExpectedOutput: &DeviceTranslator{ - Table: map[string]string{ - "/dev/nvme0n1" : "/dev/xvdf", - "/dev/xvdf": "/dev/nvme0n1", - }, - }, - ExpectedErr: nil, - }, - { - Name: "Get DeviceTranslator for Traditional EBS Device", - DeviceService: &mockDeviceService{ - getBlockDevices: func() ([]string, error) { - return []string{"/dev/xvdf"}, nil - }, - }, - NVMeService: &mockNVMeService{ - getBlockDeviceMapping: func(device string) (string, error) { - return "", fmt.Errorf("🔴 getBlockDeviceMapping() should not be called") - }, - }, - ExpectedOutput: &DeviceTranslator{ - Table: map[string]string{ - "/dev/xvdf": "/dev/xvdf", - }, - }, - ExpectedErr: nil, - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - dts := &EbsDeviceTranslator{ - DeviceService: subtest.DeviceService, - NVMeService: subtest.NVMeService, - } - dt, err := dts.GetTranslator() - if !cmp.Equal(dt, subtest.ExpectedOutput) { - t.Errorf("GetTranslator() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, dt) - } - utils.CheckError("GetTranslator()", t, subtest.ExpectedErr, err) - }) - } -} diff --git a/internal/service/file.go b/internal/service/file.go index b5f68df..e7f184a 100644 --- a/internal/service/file.go +++ b/internal/service/file.go @@ -1,72 +1,70 @@ package service import ( + "fmt" "os" "syscall" - "fmt" + + "github.com/reecetech/ebs-bootstrap/internal/model" ) -// File Service Interface [START] +const ( + DefaultDirectoryPermissions = os.FileMode(0755) +) -type FileInfo struct { - Owner string - Group string - Permissions string - Exists bool -} +// File Service Interface [START] type FileService interface { - GetStats(file string) (*FileInfo, error) - ValidateFile(path string) (error) + GetFile(file string) (*model.File, error) + CreateDirectory(path string) error + ChangeOwner(file string, uid int, gid int) error + ChangePermissions(file string, perms model.FilePermissions) error } // File Service Interface [END] -type UnixFileService struct {} +type UnixFileService struct{} -func (ds *UnixFileService) GetStats(file string) (*FileInfo, error) { - info, err := os.Stat(file) +func NewUnixFileService() *UnixFileService { + return &UnixFileService{} +} + +func (ufs *UnixFileService) GetFile(file string) (*model.File, error) { + info, err := os.Lstat(file) if err != nil { - if os.IsNotExist(err) { - return &FileInfo{Exists: false}, nil - } - return nil, err + return nil, err } if stat, ok := info.Sys().(*syscall.Stat_t); ok { - return &FileInfo{ - Owner: fmt.Sprintf("%d", stat.Uid), - Group: fmt.Sprintf("%d", stat.Gid), - Permissions: fmt.Sprintf("%o", info.Mode().Perm()), - Exists: true, + var ft model.FileType + switch mode := info.Mode(); { + case mode.IsRegular(): + ft = model.RegularFile + case mode.IsDir(): + ft = model.Directory + default: + ft = model.Special + } + return &model.File{ + Path: file, + DeviceId: int(stat.Dev), + InodeNo: int(stat.Ino), + Uid: int(stat.Uid), + Gid: int(stat.Gid), + Permissions: model.FilePermissions(info.Mode().Perm()), + Type: ft, }, nil } return nil, fmt.Errorf("🔴 %s: Failed to get stats", file) } -func (ds *UnixFileService) ValidateFile(path string) (error) { - s, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("🔴 %s does not exist", path) - } - return err - } - if !s.Mode().IsRegular() { - return fmt.Errorf("🔴 %s is not a regular file", path) - } - return nil +func (ufs *UnixFileService) CreateDirectory(path string) error { + return os.MkdirAll(path, DefaultDirectoryPermissions) } -func (ds *UnixFileService) ValidateDirectory(path string) (error) { - s, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("🔴 %s does not exist", path) - } - return err - } - if !s.Mode().IsDir() { - return fmt.Errorf("🔴 %s is not a directory", path) - } - return nil +func (ufs *UnixFileService) ChangeOwner(file string, uid int, gid int) error { + return os.Chown(file, uid, gid) +} + +func (ufs *UnixFileService) ChangePermissions(file string, perms model.FilePermissions) error { + return os.Chmod(file, perms.Perm()) } diff --git a/internal/service/file_test.go b/internal/service/file_test.go deleted file mode 100644 index a80d90d..0000000 --- a/internal/service/file_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package service - -import ( - "fmt" - "os" - "testing" - "ebs-bootstrap/internal/utils" - "github.com/google/go-cmp/cmp" -) - -func TestGetStats(t *testing.T) { - fs := &UnixFileService{} - t.Run("Get File Stats (Existing File)", func(t *testing.T) { - owner, group, permissions := os.Getuid(), os.Getgid(), os.FileMode(0644) - f, err := os.CreateTemp("", "sample") - utils.CheckError("CreateTemp()", t, nil, err) - defer os.Remove(f.Name()) - - err = os.Chown(f.Name(), owner, group) - utils.CheckError("Chown()", t, nil, err) - - err = os.Chmod(f.Name(), permissions) - utils.CheckError("Chmod()", t, nil, err) - - actual, err := fs.GetStats(f.Name()) - utils.CheckError("GetStats()", t, nil, err) - - expected := &FileInfo{ - Owner: fmt.Sprintf("%d", owner), - Group: fmt.Sprintf("%d", group), - Permissions: fmt.Sprintf("%o", permissions), - Exists: true, - } - if !cmp.Equal(actual, expected) { - t.Errorf("GetStats() [output] mismatch: Expected=%+v Actual=%+v", expected, actual) - } - }) - t.Run("Get File Stats (Non-Existent File)", func(t *testing.T) { - expected := &FileInfo{Exists: false} - actual, err := fs.GetStats("/non-existent-file/file.txt") - if !cmp.Equal(actual, expected) { - t.Errorf("GetStats() [output] mismatch: Expected=%+v Actual=%+v", expected, actual) - } - utils.CheckError("GetStats()", t, nil, err) - }) -} - -func TestValidateFile(t *testing.T) { - fs := &UnixFileService{} - - // Create a variable to the current working directory - d, err := os.Getwd() - if err != nil { - t.Errorf("os.Getwd() [error] %s", err) - return - } - - // Create a temporary file - f, err := os.CreateTemp("", "validate-file") - if err != nil { - t.Errorf("os.CreateTemp() [error] %s", err) - return - } - defer os.Remove(f.Name()) - - subtests := []struct{ - Name string - Path string - ExpectedErr error - }{ - { - Name: "Valid (Existing File)", - Path: f.Name(), - ExpectedErr: nil, - }, - { - Name: "Invalid (Existing Directory)", - Path: d, - ExpectedErr: fmt.Errorf("🔴 %s is not a regular file", d), - }, - { - Name: "Invalid: (Non-existing File)", - Path: "/doesnt-exist", - ExpectedErr: fmt.Errorf("🔴 /doesnt-exist does not exist"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - err := fs.ValidateFile(subtest.Path) - utils.CheckError("ValidateFile()", t, subtest.ExpectedErr, err) - }) - } -} - -func TestValidateDirectory(t *testing.T) { - fs := &UnixFileService{} - - // Create a variable to the current working directory - d, err := os.Getwd() - if err != nil { - t.Errorf("os.Getwd() [error] %s", err) - return - } - - // Create a temporary file - f, err := os.CreateTemp("", "validate-directory") - if err != nil { - t.Errorf("os.CreateTemp() [error] %s", err) - return - } - defer os.Remove(f.Name()) - - subtests := []struct{ - Name string - Path string - ExpectedErr error - }{ - { - Name: "Valid (Existing Directory)", - Path: d, - ExpectedErr: nil, - }, - { - Name: "Invalid (Existing File)", - Path: f.Name(), - ExpectedErr: fmt.Errorf("🔴 %s is not a directory", f.Name()), - }, - { - Name: "Invalid (Non-existing Directory)", - Path: "/doesnt-exist", - ExpectedErr: fmt.Errorf("🔴 /doesnt-exist does not exist"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - err := fs.ValidateDirectory(subtest.Path) - utils.CheckError("ValidateDirectory()", t, subtest.ExpectedErr, err) - }) - } -} diff --git a/internal/service/filesystem.go b/internal/service/filesystem.go new file mode 100644 index 0000000..6ea8cf8 --- /dev/null +++ b/internal/service/filesystem.go @@ -0,0 +1,213 @@ +package service + +import ( + "fmt" + "regexp" + "strconv" + + "github.com/reecetech/ebs-bootstrap/internal/model" + "github.com/reecetech/ebs-bootstrap/internal/utils" +) + +type FileSystemService interface { + GetSize(name string) (uint64, error) // (bytes) + GetFileSystem() model.FileSystem + Format(name string) error + Label(name string, label string) error + Resize(name string) error + DoesResizeRequireMount() bool + DoesLabelRequireUnmount() bool +} + +type FileSystemServiceFactory interface { + Select(fs model.FileSystem) (FileSystemService, error) +} + +type LinuxFileSystemServiceFactory struct { + RunnerFactory utils.RunnerFactory +} + +func NewLinuxFileSystemServiceFactory(rc utils.RunnerFactory) *LinuxFileSystemServiceFactory { + return &LinuxFileSystemServiceFactory{ + RunnerFactory: rc, + } +} + +func (fsf *LinuxFileSystemServiceFactory) Select(fs model.FileSystem) (FileSystemService, error) { + switch fs { + case model.Ext4: + return NewExt4Service(fsf.RunnerFactory), nil + case model.Xfs: + return NewXfsService(fsf.RunnerFactory), nil + default: + return nil, fmt.Errorf("🔴 An unsupported filesystem was encountered") + } +} + +type Ext4Service struct { + RunnerFactory utils.RunnerFactory +} + +func NewExt4Service(rc utils.RunnerFactory) *Ext4Service { + return &Ext4Service{RunnerFactory: rc} +} + +func (es *Ext4Service) GetFileSystem() model.FileSystem { + return model.Ext4 +} + +func (es *Ext4Service) Format(name string) error { + r := es.RunnerFactory.Select(utils.MkfsExt4) + _, err := r.Command(name) + if err != nil { + return err + } + return nil +} + +func (es *Ext4Service) Label(name string, label string) error { + if len(label) > 16 { + return fmt.Errorf("🔴 %s: Label '%s'cannot exceed 16 characters for the ext4 file system", name, label) + } + r := es.RunnerFactory.Select(utils.E2Label) + _, err := r.Command(name, label) + if err != nil { + return err + } + return nil +} + +func (es *Ext4Service) Resize(name string) error { + r := es.RunnerFactory.Select(utils.Resize2fs) + _, err := r.Command(name) + if err != nil { + return err + } + return nil +} + +func (es *Ext4Service) GetSize(name string) (uint64, error) { + r := es.RunnerFactory.Select(utils.Tune2fs) + output, err := r.Command("-l", name) + if err != nil { + return 0, err + } + // Regex (Block Size) + rebs := regexp.MustCompile(`Block size:\s+(\d+)`) + // Match (Block Size) + mbs := rebs.FindStringSubmatch(output) + if len(mbs) != 2 { + return 0, fmt.Errorf("🔴 %s: Block size not found tune2fs output", name) + } + // String (Block Size) + sbs := mbs[1] + // Block Size + bs, err := strconv.ParseUint(sbs, 10, 64) + if err != nil { + return 0, fmt.Errorf("🔴 Failed to cast block size to unsigned 64-bit integer") + } + + // Regex (Block Count) + rebc := regexp.MustCompile(`Block count:\s+(\d+)`) + // Match (Block Count) + mbc := rebc.FindStringSubmatch(output) + if len(mbs) != 2 { + return 0, fmt.Errorf("🔴 %s: Block count not found tune2fs output", name) + } + // String (Block Count) + sbc := mbc[1] + // Block Count + bc, err := strconv.ParseUint(sbc, 10, 64) + if err != nil { + return 0, fmt.Errorf("🔴 Failed to cast block size to unsigned 64-bit integer") + } + return bs * bc, nil +} + +func (es *Ext4Service) DoesResizeRequireMount() bool { + return false +} + +func (es *Ext4Service) DoesLabelRequireUnmount() bool { + return false +} + +type XfsService struct { + RunnerFactory utils.RunnerFactory +} + +func NewXfsService(rc utils.RunnerFactory) *XfsService { + return &XfsService{RunnerFactory: rc} +} + +func (es *XfsService) GetFileSystem() model.FileSystem { + return model.Xfs +} + +func (xs *XfsService) Format(name string) error { + r := xs.RunnerFactory.Select(utils.MkfsXfs) + _, err := r.Command(name) + if err != nil { + return err + } + return nil +} + +func (xs *XfsService) Label(name string, label string) error { + if len(label) > 12 { + return fmt.Errorf("🔴 %s: Label '%s' cannot exceed 12 characters for the XFS file system", name, label) + } + r := xs.RunnerFactory.Select(utils.XfsAdmin) + _, err := r.Command("-L", label, name) + if err != nil { + return err + } + return nil +} + +func (es *XfsService) Resize(name string) error { + r := es.RunnerFactory.Select(utils.XfsGrowfs) + _, err := r.Command(name) + if err != nil { + return err + } + return nil +} + +func (xs *XfsService) GetSize(name string) (uint64, error) { + r := xs.RunnerFactory.Select(utils.XfsInfo) + output, err := r.Command(name) + if err != nil { + return 0, err + } + // Regex (Data) + red := regexp.MustCompile(`data\s+=\s+bsize=(\d+)\s+blocks=(\d+)`) + // Match (Data) + md := red.FindStringSubmatch(output) + if len(md) != 3 { + return 0, fmt.Errorf("🔴 %s: Block size and block count not found xfs_info output", name) + } + // String (Block Size) + sbs := md[1] + // Block Size + bs, err := strconv.ParseUint(sbs, 10, 64) + if err != nil { + return 0, fmt.Errorf("🔴 Failed to cast block size to unsigned 64-bit integer") + } + // String (Block Count) + sbc := md[2] + // Block Count + bc, err := strconv.ParseUint(sbc, 10, 64) + if err != nil { + return 0, fmt.Errorf("🔴 Failed to cast block count to unsigned 64-bit integer") + } + return bs * bc, nil +} + +func (es *XfsService) DoesResizeRequireMount() bool { + return true +} + +func (es *XfsService) DoesLabelRequireUnmount() bool { + return true +} diff --git a/internal/service/nvme.go b/internal/service/nvme.go index 10cd361..c92e248 100644 --- a/internal/service/nvme.go +++ b/internal/service/nvme.go @@ -1,166 +1,215 @@ package service import ( - "fmt" - "os" - "syscall" - "unsafe" - "strings" - "unicode" + "fmt" + "os" + "regexp" + "strings" + "syscall" + "unsafe" ) const ( - NVME_ADMIN_IDENTIFY = 0x06 - NVME_IOCTL_ADMIN_CMD = 0xC0484E41 - AMZN_NVME_VID = 0x1D0F - AMZN_NVME_EBS_MN = "Amazon Elastic Block Store" + NVME_ADMIN_IDENTIFY = 0x06 + NVME_IOCTL_ADMIN_CMD = 0xC0484E41 + AMZN_NVME_VID = 0x1D0F + AMZN_NVME_EBS_MN = "Amazon Elastic Block Store" + AMZN_NVME_INS_MN = "Amazon EC2 NVMe Instance Storage" ) type nvmeAdminCommand struct { - Opcode uint8 - Flags uint8 - Cid uint16 - Nsid uint32 - Reserved0 uint64 - Mptr uint64 - Addr uint64 - Mlen uint32 - Alen uint32 - Cdw10 uint32 - Cdw11 uint32 - Cdw12 uint32 - Cdw13 uint32 - Cdw14 uint32 - Cdw15 uint32 - Reserved1 uint64 + Opcode uint8 + Flags uint8 + Cid uint16 + Nsid uint32 + Reserved0 uint64 + Mptr uint64 + Addr uint64 + Mlen uint32 + Alen uint32 + Cdw10 uint32 + Cdw11 uint32 + Cdw12 uint32 + Cdw13 uint32 + Cdw14 uint32 + Cdw15 uint32 + Reserved1 uint64 } type nvmeIdentifyControllerAmznVS struct { - Bdev [32]byte - Reserved0 [1024 - 32]byte + Bdev [32]byte + Reserved0 [1024 - 32]byte } type nvmeIdentifyControllerPSD struct { - Mp uint16 - Reserved0 uint16 - Enlat uint32 - Exlat uint32 - Rrt uint8 - Rrl uint8 - Rwt uint8 - Rwl uint8 - Reserved1 [16]byte + Mp uint16 + Reserved0 uint16 + Enlat uint32 + Exlat uint32 + Rrt uint8 + Rrl uint8 + Rwt uint8 + Rwl uint8 + Reserved1 [16]byte } type nvmeIdentifyController struct { - Vid uint16 - Ssvid uint16 - Sn [20]byte - Mn [40]byte - Fr [8]byte - Rab uint8 - Ieee [3]uint8 - Mic uint8 - Mdts uint8 - Reserved0 [256 - 78]byte - Oacs uint16 - Acl uint8 - Aerl uint8 - Frmw uint8 - Lpa uint8 - Elpe uint8 - Npss uint8 - Avscc uint8 - Reserved1 [512 - 265]byte - Sqes uint8 - Cqes uint8 - Reserved2 uint16 - Nn uint32 - Oncs uint16 - Fuses uint16 - Fna uint8 - Vwc uint8 - Awun uint16 - Awupf uint16 - Nvscc uint8 - Reserved3 [704 - 531]byte - Reserved4 [2048 - 704]byte - Psd [32]nvmeIdentifyControllerPSD - Vs nvmeIdentifyControllerAmznVS + Vid uint16 + Ssvid uint16 + Sn [20]byte + Mn [40]byte + Fr [8]byte + Rab uint8 + Ieee [3]uint8 + Mic uint8 + Mdts uint8 + Reserved0 [256 - 78]byte + Oacs uint16 + Acl uint8 + Aerl uint8 + Frmw uint8 + Lpa uint8 + Elpe uint8 + Npss uint8 + Avscc uint8 + Reserved1 [512 - 265]byte + Sqes uint8 + Cqes uint8 + Reserved2 uint16 + Nn uint32 + Oncs uint16 + Fuses uint16 + Fna uint8 + Vwc uint8 + Awun uint16 + Awupf uint16 + Nvscc uint8 + Reserved3 [704 - 531]byte + Reserved4 [2048 - 704]byte + Psd [32]nvmeIdentifyControllerPSD + Vs nvmeIdentifyControllerAmznVS } type NVMeDevice struct { - Name string - IdCtrl nvmeIdentifyController + Name string + IdCtrl nvmeIdentifyController } func NewNVMeDevice(name string) (*NVMeDevice, error) { - d := &NVMeDevice{Name: name} - if err := d.nvmeIOctl(); err != nil { - return nil, err - } - return d, nil + d := &NVMeDevice{Name: name} + if err := d.NVMeIOctl(); err != nil { + return nil, err + } + return d, nil } -func (d *NVMeDevice) nvmeIOctl() error { - idResponse := uintptr(unsafe.Pointer(&d.IdCtrl)) - idLen := unsafe.Sizeof(d.IdCtrl) - - adminCmd := nvmeAdminCommand{ - Opcode: NVME_ADMIN_IDENTIFY, - Addr: uint64(idResponse), - Alen: uint32(idLen), - Cdw10: 1, - } - - nvmeFile, err := os.OpenFile(d.Name, os.O_RDONLY, 0) - if err != nil { - return err - } - defer nvmeFile.Close() - - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, nvmeFile.Fd(), NVME_IOCTL_ADMIN_CMD, uintptr(unsafe.Pointer(&adminCmd))) - if errno != 0 { - return fmt.Errorf("🔴 ioctl error: %v", errno) - } - - return nil +func (d *NVMeDevice) NVMeIOctl() error { + idResponse := uintptr(unsafe.Pointer(&d.IdCtrl)) + idLen := unsafe.Sizeof(d.IdCtrl) + + adminCmd := nvmeAdminCommand{ + Opcode: NVME_ADMIN_IDENTIFY, + Addr: uint64(idResponse), + Alen: uint32(idLen), + Cdw10: 1, + } + + nvmeFile, err := os.OpenFile(d.Name, os.O_RDONLY, 0) + if err != nil { + return err + } + defer nvmeFile.Close() + + _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, nvmeFile.Fd(), NVME_IOCTL_ADMIN_CMD, uintptr(unsafe.Pointer(&adminCmd))) + if errno != 0 { + return fmt.Errorf("🔴 ioctl error: %v", errno) + } + + return nil } // NVMe Service [Start] - type NVMeService interface { - GetBlockDeviceMapping(device string) (string, error) + GetBlockDeviceMapping(device string) (string, error) } // NVMe Service [END] -type AwsNVMeService struct {} +type AwsNitroNVMeService struct{} + +func NewAwsNitroNVMeService() *AwsNitroNVMeService { + return &AwsNitroNVMeService{} +} + +func (ns *AwsNitroNVMeService) GetBlockDeviceMapping(device string) (string, error) { + nd, err := NewNVMeDevice(device) + if err != nil { + return "", err + } + return ns.getBlockDeviceMapping(nd) +} + +func (ns *AwsNitroNVMeService) isEBSVolume(nd *NVMeDevice) bool { + vid := nd.IdCtrl.Vid + mn := strings.TrimRightFunc(string(nd.IdCtrl.Mn[:]), ns.TrimModelNumber) + return vid == AMZN_NVME_VID && mn == AMZN_NVME_EBS_MN +} + +func (ns *AwsNitroNVMeService) isInstanceStoreVolume(nd *NVMeDevice) bool { + vid := nd.IdCtrl.Vid + mn := strings.TrimRightFunc(string(nd.IdCtrl.Mn[:]), ns.TrimModelNumber) + return vid == AMZN_NVME_VID && mn == AMZN_NVME_INS_MN +} -func (ns *AwsNVMeService) GetBlockDeviceMapping(device string) (string, error) { - nd, err := NewNVMeDevice(device); - if err != nil { - return "", err - } - return ns.getBlockDeviceMapping(nd) +func (ns *AwsNitroNVMeService) getBlockDeviceMapping(nd *NVMeDevice) (string, error) { + var bdm string + if ns.isEBSVolume(nd) { + bdm = strings.TrimRightFunc(string(nd.IdCtrl.Vs.Bdev[:]), ns.TrimVendorSpecfic) + } + if ns.isInstanceStoreVolume(nd) { + // Vendor Specfic (vs) + vs := strings.TrimRightFunc(string(nd.IdCtrl.Vs.Bdev[:]), ns.TrimVendorSpecfic) + // Regex Block device Mapping + rebdm := regexp.MustCompile(`ephemeral[0-9]:(sd[a-z]|none)`) + // Match Block Device Mapping + mbdm := rebdm.FindStringSubmatch(vs) + if len(mbdm) != 2 { + return "", fmt.Errorf("🔴 %s: Instance-store vendor specific metadata did not match pattern . Pattern=%s, Actual=%s", nd.Name, rebdm.String(), vs) + } + if mbdm[1] == "none" { + return "", fmt.Errorf("🔴 %s: Must provide a device name to the Instance Store NVMe block device mapping", nd.Name) + } + bdm = mbdm[1] + } + if len(bdm) == 0 { + return "", fmt.Errorf("🔴 %s is not an AWS-managed NVME device", nd.Name) + } + if !strings.HasPrefix(bdm, "/dev/") { + bdm = "/dev/" + bdm + } + return bdm, nil } -func (ns *AwsNVMeService) isEBSVolume(nd *NVMeDevice) bool { - vid := nd.IdCtrl.Vid - mn := strings.TrimRightFunc(string(nd.IdCtrl.Mn[:]), unicode.IsSpace) - return vid == AMZN_NVME_VID && mn == AMZN_NVME_EBS_MN +func (ns *AwsNitroNVMeService) TrimModelNumber(r rune) bool { + // Explanation: + // - Both the AWS EC2 and EBS team use the 0x20 (space) byte to pad out the Model Number + return r == 0x20 } -func (ns *AwsNVMeService) getBlockDeviceMapping(nd *NVMeDevice) (string, error) { - var bdm string; - if ns.isEBSVolume(nd) { - bdm = strings.TrimRightFunc(string(nd.IdCtrl.Vs.Bdev[:]), unicode.IsSpace) - } - if bdm == "" { - return "", fmt.Errorf("🔴 %s is not an AWS-managed NVME device", nd.Name) - } - if !strings.HasPrefix(bdm, "/dev/") { - bdm = "/dev/" + bdm - } - return bdm, nil +func (ns *AwsNitroNVMeService) TrimVendorSpecfic(r rune) bool { + // Explanation: + // - The AWS EC2 team uses the 0x00 (null) byte to pad out the Vendor Specific allocation + // - The AWS EBS team uses the 0x20 (space) byte to pad out the Vendor Specific allocation + // - It is frustrating that the padding character is not standardised, but we can + // work around this by checking for both bytes when trimming the Vendor Specific allocation + // Examples: + // nd.IdCtrl.Vs.Bdev[:] ("Amazon EC2 NVMe Instance Storage") + // 0 1 2 3 4 5 6 7 8 9 a b c d e f + // 0000: 65 70 68 65 6d 65 72 61 6c 30 3a 73 64 68 00 00 "ephemeral0:sdh.." + // 0010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 "................" + // nd.IdCtrl.Vs.Bdev[:] (Amazon Elastic Block Store) + // 0 1 2 3 4 5 6 7 8 9 a b c d e f + // 0000: 2f 64 65 76 2f 73 64 63 20 20 20 20 20 20 20 20 "/dev/sdc........" + // 0010: 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 "................" + return r == 0x00 || r == 0x20 } diff --git a/internal/service/nvme_test.go b/internal/service/nvme_test.go deleted file mode 100644 index 237d00e..0000000 --- a/internal/service/nvme_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package service - -import ( - "fmt" - "testing" - "ebs-bootstrap/internal/utils" -) - -const ( - UNSUPPORTED_NVME_VID = 0xFFFF - UNSUPPORTED_NVME_MN = "External NVME Manufacturer" -) - -func TestAwsNVMeService(t *testing.T) { - subtests := []struct{ - Name string - Device string - VendorId uint16 - ModelNumber string - BlockDevice string - ExpectedOutput string - ExpectedErr error - }{ - { - Name: "EBS NVMe Device (Partial Block Device)", - Device: "/dev/nvme1n1", - VendorId: AMZN_NVME_VID, - ModelNumber: AMZN_NVME_EBS_MN, - BlockDevice: "sdb", - ExpectedOutput: "/dev/sdb", - ExpectedErr: nil, - }, - { - Name: "EBS NVMe Device (Complete Block Device)", - Device: "/dev/nvme1n1", - VendorId: AMZN_NVME_VID, - ModelNumber: AMZN_NVME_EBS_MN, - BlockDevice: "/dev/sdb", - ExpectedOutput: "/dev/sdb", - ExpectedErr: nil, - }, - { - Name: "Invalid NVMe Device (Unsupported Vendor ID)", - Device: "/dev/nvme1n1", - VendorId: UNSUPPORTED_NVME_VID, - ModelNumber: AMZN_NVME_EBS_MN, - BlockDevice: "", - ExpectedOutput: "", - ExpectedErr: fmt.Errorf("🔴 /dev/nvme1n1 is not an AWS-managed NVME device"), - }, - { - Name: "Invalid NVMe Device (Unsupported Model Number)", - Device: "/dev/nvme1n1", - VendorId: AMZN_NVME_VID, - ModelNumber: UNSUPPORTED_NVME_MN, - BlockDevice: "", - ExpectedOutput: "", - ExpectedErr: fmt.Errorf("🔴 /dev/nvme1n1 is not an AWS-managed NVME device"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - nd := &NVMeDevice{ - Name: subtest.Device, - IdCtrl: nvmeIdentifyController{ - Vid: subtest.VendorId, - Mn: parseModelNumber(subtest.ModelNumber), - Vs: nvmeIdentifyControllerAmznVS{ - Bdev: parseBlockDevice(subtest.BlockDevice), - }, - }, - } - ns := &AwsNVMeService{} - bdm, err := ns.getBlockDeviceMapping(nd) - if bdm != subtest.ExpectedOutput { - t.Errorf("getBlockDeviceMapping() [output] mismatch: Expected=%+v Actual=%+v", subtest.ExpectedOutput, bdm) - } - utils.CheckError("getBlockDeviceMapping()", t, subtest.ExpectedErr, err) - }) - } -} - -func parseModelNumber(input string) [40]byte { - var mn [40]byte - copy(mn[:], input) - if len(input) < 40 { - for i := len(input); i < 40; i++ { - mn[i] = ' ' - } - } - return mn -} - -func parseBlockDevice(input string) [32]byte { - var bd [32]byte - copy(bd[:], input) - if len(input) < 32 { - for i := len(input); i < 32; i++ { - bd[i] = ' ' - } - } - return bd -} diff --git a/internal/service/owner.go b/internal/service/owner.go new file mode 100644 index 0000000..706f170 --- /dev/null +++ b/internal/service/owner.go @@ -0,0 +1,82 @@ +package service + +import ( + "fmt" + "os/user" + "strconv" + + "github.com/reecetech/ebs-bootstrap/internal/model" +) + +type OwnerService interface { + GetCurrentUser() (*model.User, error) + GetUser(owner string) (*model.User, error) + GetGroup(owner string) (*model.Group, error) +} + +type UnixOwnerService struct{} + +func NewUnixOwnerService() *UnixOwnerService { + return &UnixOwnerService{} +} + +func (uos *UnixOwnerService) GetCurrentUser() (*model.User, error) { + u, err := user.Current() + if err != nil { + return nil, fmt.Errorf("🔴 Failed to get current user") + } + uid, err := strconv.Atoi(u.Uid) + if err != nil { + return nil, fmt.Errorf("🔴 Failed to cast user (id) to integer") + } + return &model.User{ + Name: u.Name, + Uid: uid, + }, nil +} + +func (uos *UnixOwnerService) GetUser(us string) (*model.User, error) { + var u *user.User + if _, err := strconv.Atoi(us); err != nil { + // If not a valid integer, try to look up by username + u, err = user.Lookup(us) + if err != nil { + return nil, fmt.Errorf("🔴 User (name) %s does not exist", us) + } + } else { + u, err = user.LookupId(us) + if err != nil { + return nil, fmt.Errorf("🔴 User (id) %s does not exist", us) + } + } + + uid, err := strconv.Atoi(u.Uid) + if err != nil { + return nil, fmt.Errorf("🔴 Failed to cast user (id) to integer") + } + + return &model.User{Name: u.Username, Uid: uid}, nil +} + +func (uos *UnixOwnerService) GetGroup(grp string) (*model.Group, error) { + var g *user.Group + if _, err := strconv.Atoi(grp); err != nil { + // If not a valid integer, try to look up by group name + g, err = user.LookupGroup(grp) + if err != nil { + return nil, fmt.Errorf("🔴 Group (name) %s does not exist", grp) + } + } else { + g, err = user.LookupGroupId(grp) + if err != nil { + return nil, fmt.Errorf("🔴 Group (id) %s does not exist", grp) + } + } + + gid, err := strconv.Atoi(g.Gid) + if err != nil { + return nil, fmt.Errorf("🔴 Failed to cast group (id) to integer") + } + + return &model.Group{Name: g.Name, Gid: gid}, nil +} diff --git a/internal/state/device.go b/internal/state/device.go deleted file mode 100644 index 100b781..0000000 --- a/internal/state/device.go +++ /dev/null @@ -1,110 +0,0 @@ -package state - -import ( - "fmt" - "log" - "ebs-bootstrap/internal/config" - "ebs-bootstrap/internal/service" -) - -type deviceProperties struct { - Name string - Fs string - MountPoint string - Owner string - Group string - Label string - Permissions string -} - -type Device struct { - Properties deviceProperties - DeviceService service.DeviceService - FileService service.FileService -} - -func NewDevice(name string, ds service.DeviceService, fs service.FileService) (*Device, error) { - s := &Device{ - DeviceService: ds, - FileService: fs, - Properties: deviceProperties{Name: name}, - } - err := s.Pull() - if err != nil { - return nil, err - } - return s, nil -} - -func (d *Device) Pull() (error) { - name := d.Properties.Name - di, err := d.DeviceService.GetDeviceInfo(name) - if err != nil { - return err - } - p := deviceProperties{ - Name: name, - Fs: di.Fs, - Label: di.Label, - MountPoint: di.MountPoint, - } - - if p.MountPoint == "" { - log.Printf("🟡 %s: No mount-point detected. Skip further checks...", name) - d.Properties = p - return nil - } - - fi, err := d.FileService.GetStats(p.MountPoint) - if err != nil { - return err - } - if fi.Exists { - p.Owner = fi.Owner - p.Group = fi.Group - p.Permissions = fi.Permissions - } - - d.Properties = p - return nil -} - -func (d *Device) Diff(c *config.Config) (error) { - name := d.Properties.Name - if name == "" { - return fmt.Errorf("🔴 An unexpected error occured") - } - desired, found := c.Devices[name] - if !found { - return fmt.Errorf("🔴 %s: Couldn't find device in config", name) - } - - if d.Properties.Fs != string(desired.Fs) { - return fmt.Errorf("🔴 File System [%s]: Expected=%s", d.Properties.Name, desired.Fs) - } - - if d.Properties.Label != string(desired.Label) { - return fmt.Errorf("🔴 Label [%s]: Expected=%s", d.Properties.Name, desired.Label) - } - - if d.Properties.MountPoint != string(desired.MountPoint) { - return fmt.Errorf("🔴 Mount Point [%s]: Expected=%s", d.Properties.Name, desired.MountPoint) - } - - if d.Properties.Owner != string(desired.Owner) { - return fmt.Errorf("🔴 Owner [%s]: Expected=%s", d.Properties.MountPoint, desired.Owner) - } - - if d.Properties.Group != string(desired.Group) { - return fmt.Errorf("🔴 Group: [%s]: Expected=%s", d.Properties.MountPoint, desired.Group) - } - - if d.Properties.Permissions != string(desired.Permissions) { - return fmt.Errorf("🔴 Permissions [%s]: Expected=%s", d.Properties.MountPoint, desired.Permissions) - } - return nil -} - -func (d *Device) Push(c *config.Config) (error) { - return nil -} diff --git a/internal/state/device_test.go b/internal/state/device_test.go deleted file mode 100644 index 31ce0b7..0000000 --- a/internal/state/device_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package state - -import ( - "fmt" - "testing" - "ebs-bootstrap/internal/utils" - "ebs-bootstrap/internal/service" - "ebs-bootstrap/internal/config" -) - -type mockDeviceService struct { - getDeviceInfo func(device string) (*service.DeviceInfo, error) -} - -func (ds *mockDeviceService) GetBlockDevices() ([]string, error) { - return nil, fmt.Errorf("🔴 GetBlockDevices() not implemented") -} - -func (ds *mockDeviceService) GetDeviceInfo(device string) (*service.DeviceInfo, error) { - return ds.getDeviceInfo(device) -} - -type mockFileService struct { - getStats func(file string) (*service.FileInfo, error) -} - -func (fs *mockFileService) GetStats(file string) (*service.FileInfo, error) { - return fs.getStats(file) -} - -func (fs *mockFileService) ValidateFile(path string) (error) { - return fmt.Errorf("🔴 ValidateFile() not implemented") -} - -func TestDevice(t *testing.T) { - subtests := []struct { - Name string - DeviceName string - DeviceService service.DeviceService - FileService service.FileService - ExpectedErr error - }{ - { - Name: "Non-Existent Device", - DeviceName: "/dev/doesnt-exist", - DeviceService: &mockDeviceService{ - getDeviceInfo: func(device string) (*service.DeviceInfo, error) { - return nil, fmt.Errorf("🔴 /dev/doesnt-exist not a block device") - }, - }, - FileService: &mockFileService{ - getStats: func(file string) (*service.FileInfo, error) { - return nil, fmt.Errorf("🔴 getStats() should not be called") - }, - }, - ExpectedErr: fmt.Errorf("🔴 /dev/doesnt-exist not a block device"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - _, err := NewDevice(subtest.DeviceName, - subtest.DeviceService, - subtest.FileService) - utils.CheckError("NewDevice()", t, subtest.ExpectedErr, err) - }) - } -} - -func TestDeviceDiff(t *testing.T) { - subtests := []struct { - Name string - DeviceName string - DeviceService service.DeviceService - FileService service.FileService - Config *config.Config - ExpectedErr error - }{ - { - Name: "No Diff Expected With Mount-Point", - DeviceName: "/dev/nvme0n1", - DeviceService: &mockDeviceService{ - getDeviceInfo: func(device string) (*service.DeviceInfo, error) { - return &service.DeviceInfo{ - Name: "/dev/nvme0n1", - Label: "external-vol", - Fs: "xfs", - MountPoint: "/mnt/app", - }, nil - }, - }, - FileService: &mockFileService{ - getStats: func(file string) (*service.FileInfo, error) { - return &service.FileInfo{ - Owner: "100", - Group: "100", - Permissions: "755", - Exists: true, - }, nil - }, - }, - Config: &config.Config{ - Devices: map[string]config.ConfigDevice{ - "/dev/nvme0n1": config.ConfigDevice{ - Fs: "xfs", - MountPoint: "/mnt/app", - Owner: "100", - Group: "100", - Label: "external-vol", - Permissions: "755", - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "No Diff Expected Without Mount-Point", - DeviceName: "/dev/nvme0n1", - DeviceService: &mockDeviceService{ - getDeviceInfo: func(device string) (*service.DeviceInfo, error) { - return &service.DeviceInfo{ - Name: "/dev/nvme0n1", - Label: "external-vol", - Fs: "xfs", - MountPoint: "", - }, nil - }, - }, - FileService: &mockFileService{ - getStats: func(file string) (*service.FileInfo, error) { - return nil, fmt.Errorf("🔴 getStats() should not be called") - }, - }, - Config: &config.Config{ - Devices: map[string]config.ConfigDevice{ - "/dev/nvme0n1": config.ConfigDevice{ - Fs: "xfs", - Label: "external-vol", - }, - }, - }, - ExpectedErr: nil, - }, - { - Name: "Diff Suggesting Fs Change (xfs->ext4)", - DeviceName: "/dev/nvme0n1", - DeviceService: &mockDeviceService{ - getDeviceInfo: func(device string) (*service.DeviceInfo, error) { - return &service.DeviceInfo{ - Name: "/dev/nvme0n1", - Label: "external-vol", - Fs: "xfs", - MountPoint: "/mnt/app", - }, nil - }, - }, - FileService: &mockFileService{ - getStats: func(file string) (*service.FileInfo, error) { - return &service.FileInfo{ - Owner: "100", - Group: "100", - Permissions: "755", - Exists: true, - }, nil - }, - }, - Config: &config.Config{ - Devices: map[string]config.ConfigDevice{ - "/dev/nvme0n1": config.ConfigDevice{ - Fs: "ext4", - MountPoint: "/mnt/app", - Owner: "100", - Group: "100", - Label: "external-vol", - Permissions: "755", - }, - }, - }, - ExpectedErr: fmt.Errorf("🔴 File System [/dev/nvme0n1]: Expected=ext4"), - }, - } - for _, subtest := range subtests { - t.Run(subtest.Name, func(t *testing.T) { - d, err := NewDevice(subtest.DeviceName, - subtest.DeviceService, - subtest.FileService) - if err != nil { - t.Errorf("NewDevice() [error] %s", err) - } - err = d.Diff(subtest.Config) - utils.CheckError("Diff()", t, subtest.ExpectedErr, err) - }) - } -} diff --git a/internal/state/state.go b/internal/state/state.go deleted file mode 100644 index a93e46c..0000000 --- a/internal/state/state.go +++ /dev/null @@ -1,11 +0,0 @@ -package state - -import ( - "ebs-bootstrap/internal/config" -) - -type State interface { - Pull() (error) - Diff(c *config.Config) (error) - Push(c *config.Config) (error) -} diff --git a/internal/utils/exec.go b/internal/utils/exec.go index 27e7829..4859848 100644 --- a/internal/utils/exec.go +++ b/internal/utils/exec.go @@ -1,29 +1,90 @@ package utils import ( + "fmt" "os/exec" "strings" - "fmt" ) +type Binary string + +const ( + Lsblk Binary = "lsblk" + MkfsExt4 Binary = "mkfs.ext4" + E2Label Binary = "e2label" + MkfsXfs Binary = "mkfs.xfs" + XfsAdmin Binary = "xfs_admin" + Mount Binary = "mount" + Umount Binary = "umount" + BlockDev Binary = "blockdev" + Tune2fs Binary = "tune2fs" + XfsInfo Binary = "xfs_info" + Resize2fs Binary = "resize2fs" + XfsGrowfs Binary = "xfs_growfs" +) + +type RunnerFactory interface { + Select(binary Binary) Runner +} + +type ExecRunnerFactory struct { + runners map[Binary]Runner +} + +func NewExecRunnerFactory() *ExecRunnerFactory { + return &ExecRunnerFactory{ + runners: map[Binary]Runner{}, + } +} + +// Caching behaviour is implemented for ExecRunnerFactory as we +// do not need to validate an ExecRunner more than once +func (rc *ExecRunnerFactory) Select(binary Binary) Runner { + r, exists := rc.runners[binary] + if !exists { + r = NewExecRunner(binary) + rc.runners[binary] = r + } + return r +} + type Runner interface { - Command(name string, arg ...string) (string, error) + Command(arg ...string) (string, error) } type ExecRunner struct { - Runner func(name string, arg ...string) *exec.Cmd + binary Binary + command func(name string, arg ...string) *exec.Cmd + lookPath func(file string) (string, error) + isValidated bool } -func NewExecRunner() *ExecRunner { - return &ExecRunner{exec.Command} +func NewExecRunner(binary Binary) *ExecRunner { + return &ExecRunner{ + binary: binary, + command: exec.Command, + lookPath: exec.LookPath, + isValidated: false, + } } -func (er *ExecRunner) Command(name string, arg ...string) (string, error) { - cmd := er.Runner(name, arg...) +func (er *ExecRunner) Command(arg ...string) (string, error) { + if !er.isValid() { + return "", fmt.Errorf("🔴 %s is either not installed or accessible from $PATH", string(er.binary)) + } + cmd := er.command(string(er.binary), arg...) o, err := cmd.CombinedOutput() output := strings.TrimRight(string(o), "\n") if err != nil { - return "", fmt.Errorf("%s: %s", err, output) + return "", fmt.Errorf("🔴 %s: %s", err, output) } return output, err } + +func (er *ExecRunner) isValid() bool { + if !er.isValidated { + _, err := er.lookPath(string(er.binary)) + er.isValidated = err == nil + } + return er.isValidated +} diff --git a/internal/utils/string.go b/internal/utils/string.go new file mode 100644 index 0000000..5f0f8af --- /dev/null +++ b/internal/utils/string.go @@ -0,0 +1,8 @@ +package utils + +func SafeString(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/internal/utils/testing.go b/internal/utils/testing.go deleted file mode 100644 index 9bbe052..0000000 --- a/internal/utils/testing.go +++ /dev/null @@ -1,37 +0,0 @@ -package utils - -import ( - "fmt" - "strings" - "os/user" - "testing" -) - -func CheckError(name string, t *testing.T, expected error, actual error) { - if actual != nil { - if expected == nil { - t.Errorf("%s [error] undetected: Actual=%v", name, actual) - return - } - if expected.Error() != actual.Error() { - t.Errorf("%s [error] mismatch: Expected=%v Actual=%v", name, expected, actual) - } - } -} - -func GetCurrentUserGroup() (*user.User, *user.Group, error) { - u, err := user.Current() - if err != nil { - return nil, nil, fmt.Errorf("🔴 Failed to get current user") - } - g, err := user.LookupGroupId(u.Gid) - if err != nil { - return nil, nil, fmt.Errorf("🔴 Failed to get current group") - } - /* user.Current() -> From experience, this function can return a username - in a capital case. This is not valid UNIX format for usernames so force - to lowercase */ - u.Name = strings.ToLower(u.Name) - g.Name = strings.ToLower(g.Name) - return u, g, nil -}