Instance volume configuration through disk device

doc/api-extensions.md

@@ -2298,3 +2298,11 @@ This adds the fields `Name` and `Project` to `lifecycle` events.
 
 This introduces a new per-NIC `limits.priority` option that works with both cgroup1 and cgroup2 unlike the deprecated `limits.network.priority` instance setting, which only worked with cgroup1.
 
+## `disk_initial_volume_configuration`
+
+This API extension provides the capability to set initial volume configurations for instance root devices.
+Initial volume configurations are prefixed with `initial.` and can be specified either through profiles or directly
+during instance initialization using the `--device` flag.
+
+Note that these configuration are applied only at the time of instance creation and subsequent modifications have
+no effect on existing devices.

doc/reference/devices_disk.md

@@ -70,6 +70,24 @@ VM `cloud-init`
 
       lxc config device add <instance_name> <device_name> disk source=cloud-init:config
 
+(devices-disk-initial-config)=
+## Initial volume configuration for instance root disk devices
+
+Initial volume configuration allows setting specific configurations for the root disk devices of new instances.
+These settings are prefixed with `initial.` and are only applied when the instance is created.
+This method allows creating instances that have unique configurations, independent of the default storage pool settings.
+
+For example, you can add an initial volume configuration for `zfs.block_mode` to an existing profile, and this
+will then take effect for each new instance you create using this profile:
+
+    lxc profile device set <profile_name> <device_name> initial.zfs.block_mode=true
+
+You can also set an initial configuration directly when creating an instance. For example:
+
+    lxc init <image> <instance_name> --device <device_name>,initial.zfs.block_mode=true
+
+Note that you cannot use initial volume configurations with custom volume options or to set the volume's size.
+
 ## Device options
 
 `disk` devices have the following device options:
@@ -79,6 +97,7 @@ Key                 | Type      | Default   | Required  | Description
 `boot.priority`     | integer   | -         | no        | Boot priority for VMs (higher value boots first)
 `ceph.cluster_name` | string    | `ceph`    | no        | The cluster name of the Ceph cluster (required for Ceph or CephFS sources)
 `ceph.user_name`    | string    | `admin`   | no        | The user name of the Ceph cluster (required for Ceph or CephFS sources)
+`initial.*`         | n/a       | -         | no        | {ref}`devices-disk-initial-config` that allows setting unique configurations independent of default storage pool settings
 `io.cache`          | string    | `none`    | no        | Only for VMs: Override the caching mode for the device (`none`, `writeback` or `unsafe`)
 `limits.max`        | string    | -         | no        | I/O limit in byte/s or IOPS for both read and write (same as setting both `limits.read` and `limits.write`)
 `limits.read`       | string    | -         | no        | I/O limit in byte/s (various suffixes supported, see {ref}`instances-limit-units`) or in IOPS (must be suffixed with `iops`) - see also {ref}`storage-configure-IO`

lxd/device/config/devices.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+
+	"github.com/canonical/lxd/shared/api"
 )
 
 // Device represents a LXD container device.
@@ -44,11 +46,16 @@ func (device Device) Validate(rules map[string]func(value string) error) error {
 			continue
 		}
 
-		// Allow user.XYZ.
+		// Allow user.* configuration.
 		if strings.HasPrefix(k, "user.") {
 			continue
 		}
 
+		// Allow initial.* configuration.
+		if strings.HasPrefix(k, "initial.") {
+			continue
+		}
+
 		if k == "nictype" && (device["type"] == "nic" || device["type"] == "infiniband") {
 			continue
 		}
@@ -82,6 +89,35 @@ func NewDevices(nativeSet map[string]map[string]string) Devices {
 	return newDevices
 }
 
+// ApplyDeviceInitialValues applies a profile initial values to root disk devices.
+func ApplyDeviceInitialValues(devices Devices, profiles []api.Profile) Devices {
+	for _, p := range profiles {
+		for devName, devConfig := range p.Devices {
+			// Apply only root disk device from profile devices to instance devices.
+			if devConfig["type"] != "disk" || devConfig["path"] != "/" || devConfig["source"] != "" {
+				continue
+			}
+
+			// Skip profile devices that are already present in the map of devices
+			// because those devices should be already populated.
+			_, ok := devices[devName]
+			if ok {
+				continue
+			}
+
+			// If profile device contains an initial.* key, add it to the map of devices.
+			for k := range devConfig {
+				if strings.HasPrefix(k, "initial.") {
+					devices[devName] = devConfig
+					break
+				}
+			}
+		}
+	}
+
+	return devices
+}
+
 // Contains checks if a given device exists in the set and if it's identical to that provided.
 func (list Devices) Contains(k string, d Device) bool {
 	// If it didn't exist, it's different

lxd/device/disk.go

@@ -349,6 +349,42 @@ func (d *disk) validateConfig(instConf instance.ConfigReader) error {
 					return fmt.Errorf("Custom filesystem volumes require a path to be defined")
 				}
 			}
+
+			// Extract initial configuration from the profile and validate them against appropriate
+			// storage driver. Currently initial configuration is only applicable to root disk devices.
+			initialConfig := make(map[string]string)
+			for k, v := range d.config {
+				prefix, newKey, found := strings.Cut(k, "initial.")
+				if found && prefix == "" {
+					initialConfig[newKey] = v
+				}
+			}
+
+			if len(initialConfig) > 0 {
+				if !shared.IsRootDiskDevice(d.config) {
+					return fmt.Errorf("Non-root disk device cannot contain initial.* configuration")
+				}
+
+				volumeType, err := storagePools.InstanceTypeToVolumeType(d.inst.Type())
+				if err != nil {
+					return err
+				}
+
+				// Create temporary volume definition.
+				vol := storageDrivers.NewVolume(
+					d.pool.Driver(),
+					d.pool.Name(),
+					volumeType,
+					storagePools.InstanceContentType(d.inst),
+					d.name,
+					initialConfig,
+					d.pool.Driver().Config())
+
+				err = d.pool.Driver().ValidateVolume(vol, true)
+				if err != nil {
+					return fmt.Errorf("Invalid initial device configuration: %v", err)
+				}
+			}
 		}
 	}
 

lxd/instance/drivers/driver_lxc.go

@@ -4205,6 +4205,33 @@ func (d *lxc) Update(args db.InstanceArgs, userRequested bool) error {
 		return newDevType.UpdatableFields(oldDevType)
 	})
 
+	// Prevent adding or updating device initial configuration.
+	if shared.StringPrefixInSlice("initial.", allUpdatedKeys) {
+		for devName, newDev := range addDevices {
+			for k, newVal := range newDev {
+				if !strings.HasPrefix(k, "initial.") {
+					continue
+				}
+
+				oldDev, ok := removeDevices[devName]
+				if !ok {
+					return fmt.Errorf("New device with initial configuration cannot be added once the instance is created")
+				}
+
+				oldVal, ok := oldDev[k]
+				if !ok {
+					return fmt.Errorf("Device initial configuration cannot be added once the instance is created")
+				}
+
+				// If newVal is an empty string it means the initial configuration
+				// has been removed.
+				if newVal != "" && newVal != oldVal {
+					return fmt.Errorf("Device initial configuration cannot be modified once the instance is created")
+				}
+			}
+		}
+	}
+
 	if userRequested {
 		// Look for deleted idmap keys.
 		protectedKeys := []string{

lxd/instance/drivers/driver_qemu.go

@@ -5071,6 +5071,33 @@ func (d *qemu) Update(args db.InstanceArgs, userRequested bool) error {
 		return newDevType.UpdatableFields(oldDevType)
 	})
 
+	// Prevent adding or updating device initial configuration.
+	if shared.StringPrefixInSlice("initial.", allUpdatedKeys) {
+		for devName, newDev := range addDevices {
+			for k, newVal := range newDev {
+				if !strings.HasPrefix(k, "initial.") {
+					continue
+				}
+
+				oldDev, ok := removeDevices[devName]
+				if !ok {
+					return fmt.Errorf("New device with initial configuration cannot be added once the instance is created")
+				}
+
+				oldVal, ok := oldDev[k]
+				if !ok {
+					return fmt.Errorf("Device initial configuration cannot be added once the instance is created")
+				}
+
+				// If newVal is an empty string it means the initial configuration
+				// has been removed.
+				if newVal != "" && newVal != oldVal {
+					return fmt.Errorf("Device initial configuration cannot be modified once the instance is created")
+				}
+			}
+		}
+	}
+
 	if userRequested {
 		// Do some validation of the config diff (allows mixed instance types for profiles).
 		err = instance.ValidConfig(d.state.OS, d.expandedConfig, true, instancetype.Any)

lxd/instances_post.go

@@ -88,12 +88,14 @@ func createFromImage(s *state.State, r *http.Request, p api.Project, profiles []
 	}
 
 	run := func(op *operations.Operation) error {
+		devices := deviceConfig.NewDevices(req.Devices)
+
 		args := db.InstanceArgs{
 			Project:     p.Name,
 			Config:      req.Config,
 			Type:        dbType,
 			Description: req.Description,
-			Devices:     deviceConfig.NewDevices(req.Devices),
+			Devices:     deviceConfig.ApplyDeviceInitialValues(devices, profiles),
 			Ephemeral:   req.Ephemeral,
 			Name:        req.Name,
 			Profiles:    profiles,
@@ -146,12 +148,14 @@ func createFromNone(s *state.State, r *http.Request, projectName string, profile
 		return response.BadRequest(err)
 	}
 
+	devices := deviceConfig.NewDevices(req.Devices)
+
 	args := db.InstanceArgs{
 		Project:     projectName,
 		Config:      req.Config,
 		Type:        dbType,
 		Description: req.Description,
-		Devices:     deviceConfig.NewDevices(req.Devices),
+		Devices:     deviceConfig.ApplyDeviceInitialValues(devices, profiles),
 		Ephemeral:   req.Ephemeral,
 		Name:        req.Name,
 		Profiles:    profiles,