Add possibility to download IPFS images

examples/default.yaml

@@ -39,9 +39,11 @@ images:
 - location: "https://cloud-images.ubuntu.com/releases/24.04/release-20240821/ubuntu-24.04-server-cloudimg-amd64.img"
   arch: "x86_64"
   digest: "sha256:0e25ca6ee9f08ec5d4f9910054b66ae7163c6152e81a3e67689d89bd6e4dfa69"
+  cid: "bafybeievi2i673t7kgzx6vsxc6lod3bvagzc364e6kes43p6catfowgone"
 - location: "https://cloud-images.ubuntu.com/releases/24.04/release-20240821/ubuntu-24.04-server-cloudimg-arm64.img"
   arch: "aarch64"
   digest: "sha256:5ecac6447be66a164626744a87a27fd4e6c6606dc683e0a233870af63df4276a"
+  cid: "bafybeich5idqhqtdh3f4nxq6kv6l2fd2nsvynha3vdqek4a7mu7nawp6sm"
 - location: "https://cloud-images.ubuntu.com/releases/24.04/release-20240821/ubuntu-24.04-server-cloudimg-riscv64.img"
   arch: "riscv64"
   digest: "sha256:f5886ad4e405e689585dfef0e96c31b06478e0cb12bc7f3fae965759a32d729e"
@@ -310,6 +312,12 @@ rosetta:
 # 🟢 Builtin default: use name from /etc/timezone or deduce from symlink target of /etc/localtime
 timezone: null
 
+# Allow using IPFS for downloading files with a provided CID.
+# If set to `true`, this will try to use ipfs cid before the location url.
+# It requires either the "ipfs" tool (kubo), or setting the IPFS_GATEWAY.
+# 🟢 Builtin default: false
+ipfs: null
+
 firmware:
   # Use legacy BIOS instead of UEFI. Ignored for aarch64 and vz.
   # 🟢 Builtin default: false

pkg/downloader/downloader.go

@@ -12,6 +12,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -41,6 +42,7 @@ const (
 	StatusDownloaded Status = "downloaded"
 	StatusSkipped    Status = "skipped"
 	StatusUsedCache  Status = "used-cache"
+	StatusUsedIPFS   Status = "used-ipfs"
 )
 
 type Result struct {
@@ -56,6 +58,8 @@ type options struct {
 	decompress     bool   // default: false (keep compression)
 	description    string // default: url
 	expectedDigest digest.Digest
+	ipfs           bool
+	cid            string
 }
 
 type Opt func(*options) error
@@ -122,6 +126,20 @@ func WithExpectedDigest(expectedDigest digest.Digest) Opt {
 	}
 }
 
+func WithIPFS(ipfs bool) Opt {
+	return func(o *options) error {
+		o.ipfs = ipfs
+		return nil
+	}
+}
+
+func WithContentIdentifier(cid string) Opt {
+	return func(o *options) error {
+		o.cid = cid
+		return nil
+	}
+}
+
 func readFile(path string) string {
 	if path == "" {
 		return ""
@@ -271,8 +289,20 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,
 	if err := writeFirst(shadURL, []byte(remote), 0o644); err != nil {
 		return nil, err
 	}
-	if err := downloadHTTP(ctx, shadData, shadTime, shadType, remote, o.description, o.expectedDigest); err != nil {
-		return nil, err
+	status := StatusDownloaded
+	if o.ipfs && o.cid != "" {
+		if err := downloadIPFS(ctx, shadData, fmt.Sprintf("ipfs://%s", o.cid), o.description, o.expectedDigest); err == nil {
+			status = StatusUsedIPFS
+		}
+	}
+	if IsIPFS(remote) {
+		if err := downloadIPFS(ctx, shadData, remote, o.description, o.expectedDigest); err != nil {
+			return nil, err
+		}
+	} else if status != StatusUsedIPFS {
+		if err := downloadHTTP(ctx, shadData, shadTime, shadType, remote, o.description, o.expectedDigest); err != nil {
+			return nil, err
+		}
 	}
 	if shadDigest != "" && o.expectedDigest != "" {
 		if err := writeFirst(shadDigest, []byte(o.expectedDigest.String()), 0o644); err != nil {
@@ -284,7 +314,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,
 		return nil, err
 	}
 	res := &Result{
-		Status:          StatusDownloaded,
+		Status:          status,
 		CachePath:       shadData,
 		LastModified:    readTime(shadTime),
 		ContentType:     readFile(shadType),
@@ -370,6 +400,10 @@ func IsLocal(s string) bool {
 	return !strings.Contains(s, "://") || strings.HasPrefix(s, "file://")
 }
 
+func IsIPFS(s string) bool {
+	return strings.HasPrefix(s, "ipfs://")
+}
+
 // canonicalLocalPath canonicalizes the local path string.
 //   - Make sure the file has no scheme, or the `file://` scheme
 //   - If it has the `file://` scheme, strip the scheme and make sure the filename is absolute
@@ -617,7 +651,64 @@ func downloadHTTP(ctx context.Context, localPath, lastModified, contentType, url
 		}
 	}
 	defer resp.Body.Close()
-	bar, err := progressbar.New(resp.ContentLength)
+	return download(resp.Body, resp.ContentLength, localPath, url, description, expectedDigest)
+}
+
+func downloadIPFS(ctx context.Context, localPath, url, description string, expectedDigest digest.Digest) error {
+	if localPath == "" {
+		return fmt.Errorf("downloadIPFS: got empty localPath")
+	}
+	logrus.Debugf("downloading %q into %q", url, localPath)
+
+	address := strings.Replace(url, "ipfs://", "", 1)
+
+	// Possibly use an ipfs getway such as "https://ipfs.io" or "http://127.0.0.1:8080"
+	if gateway := os.Getenv("IPFS_GATEWAY"); gateway != "" {
+		if strings.HasPrefix(gateway, "http") {
+			url = fmt.Sprintf("%s/ipfs/%s", gateway, address)
+			return downloadHTTP(ctx, localPath, "", "", url, description, expectedDigest)
+		}
+		return fmt.Errorf("unknown gateway: %q", gateway)
+	}
+
+	address = strings.Split(address, "/")[0] // remove file name from path
+
+	cmd := exec.CommandContext(ctx, "ipfs", "ls", address)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+	size := int64(0)
+	for _, line := range strings.Split(string(out), "\n") {
+		// Hash Size Name
+		f := strings.Fields(line)
+		if len(f) >= 2 {
+			s, err := strconv.Atoi(f[1])
+			if err != nil {
+				return err
+			}
+			size += int64(s)
+		}
+	}
+
+	cmd = exec.CommandContext(ctx, "ipfs", "cat", "--progress=false", address)
+	cmd.Stderr = os.Stderr
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	if err := download(stdout, size, localPath, url, description, expectedDigest); err != nil {
+		return err
+	}
+	return cmd.Wait()
+}
+
+func download(reader io.Reader, size int64, localPath, url, description string, expectedDigest digest.Digest) error {
+	bar, err := progressbar.New(size)
 	if err != nil {
 		return err
 	}
@@ -654,7 +745,7 @@ func downloadHTTP(ctx context.Context, localPath, lastModified, contentType, url
 		fmt.Fprintf(os.Stderr, "Downloading %s\n", description)
 	}
 	bar.Start()
-	if _, err := io.Copy(multiWriter, bar.NewProxyReader(resp.Body)); err != nil {
+	if _, err := io.Copy(multiWriter, bar.NewProxyReader(reader)); err != nil {
 		return err
 	}
 	bar.Finish()

pkg/fileutils/download.go

@@ -15,7 +15,7 @@ import (
 var ErrSkipped = errors.New("skipped to download")
 
 // DownloadFile downloads a file to the cache, optionally copying it to the destination. Returns path in cache.
-func DownloadFile(ctx context.Context, dest string, f limayaml.File, decompress bool, description string, expectedArch limayaml.Arch) (string, error) {
+func DownloadFile(ctx context.Context, dest string, f limayaml.File, decompress, ipfs bool, description string, expectedArch limayaml.Arch) (string, error) {
 	if f.Arch != expectedArch {
 		return "", fmt.Errorf("%w: %q: unsupported arch: %q", ErrSkipped, f.Location, f.Arch)
 	}
@@ -26,6 +26,8 @@ func DownloadFile(ctx context.Context, dest string, f limayaml.File, decompress
 		downloader.WithDecompress(decompress),
 		downloader.WithDescription(fmt.Sprintf("%s (%s)", description, path.Base(f.Location))),
 		downloader.WithExpectedDigest(f.Digest),
+		downloader.WithIPFS(ipfs),
+		downloader.WithContentIdentifier(f.Cid),
 	)
 	if err != nil {
 		return "", fmt.Errorf("failed to download %q: %w", f.Location, err)
@@ -36,6 +38,8 @@ func DownloadFile(ctx context.Context, dest string, f limayaml.File, decompress
 		logrus.Infof("Downloaded %s from %q", description, f.Location)
 	case downloader.StatusUsedCache:
 		logrus.Infof("Using cache %q", res.CachePath)
+	case downloader.StatusUsedIPFS:
+		logrus.Infof("Used ipfs %q", f.Cid)
 	default:
 		logrus.Warnf("Unexpected result from downloader.Download(): %+v", res)
 	}

pkg/instance/start.go

@@ -54,7 +54,7 @@ func ensureNerdctlArchiveCache(ctx context.Context, y *limayaml.LimaYAML, create
 				return path, nil
 			}
 		}
-		path, err := fileutils.DownloadFile(ctx, "", f, false, "the nerdctl archive", *y.Arch)
+		path, err := fileutils.DownloadFile(ctx, "", f, false, *y.IPFS, "the nerdctl archive", *y.Arch)
 		if err != nil {
 			errs[i] = err
 			continue

pkg/limayaml/containerd.yaml

@@ -2,8 +2,10 @@ archives:
 - location: https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-full-1.7.6-linux-amd64.tar.gz
   arch: x86_64
   digest: sha256:2c841e097fcfb5a1760bd354b3778cb695b44cd01f9f271c17507dc4a0b25606
+  cid: bafybeieipdaxd3fzy3j7syzzxdaqxramk65j7ajzcqgmi6b5jyq4jgbwue
 - location: https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-full-1.7.6-linux-arm64.tar.gz
   arch: aarch64
   digest: sha256:77c747f09853ee3d229d77e8de0dd3c85622537d82be57433dc1fca4493bab95
+  cid: bafybeibptqoynryoxytxffgyuqjnmwpuujwp6wlps4spygapuwphzkeieq
 # No arm-v7
 # No riscv64

pkg/limayaml/defaults.go

@@ -333,6 +333,16 @@ func FillDefault(y, d, o *LimaYAML, filePath string) {
 		y.TimeZone = ptr.Of(hostTimeZone())
 	}
 
+	if y.IPFS == nil {
+		y.IPFS = d.IPFS
+	}
+	if o.IPFS != nil {
+		y.IPFS = o.IPFS
+	}
+	if y.IPFS == nil {
+		y.IPFS = ptr.Of(false)
+	}
+
 	if y.SSH.LocalPort == nil {
 		y.SSH.LocalPort = d.SSH.LocalPort
 	}

pkg/limayaml/defaults_test.go

@@ -86,6 +86,7 @@ func TestFillDefault(t *testing.T) {
 			ForwardX11Trusted: ptr.Of(false),
 		},
 		TimeZone: ptr.Of(hostTimeZone()),
+		IPFS:     ptr.Of(false),
 		Firmware: Firmware{
 			LegacyBIOS: ptr.Of(false),
 		},
@@ -175,6 +176,7 @@ func TestFillDefault(t *testing.T) {
 			},
 		},
 		TimeZone: ptr.Of("Antarctica/Troll"),
+		IPFS:     ptr.Of(true),
 		Firmware: Firmware{
 			LegacyBIOS: ptr.Of(false),
 			Images: []FileWithVMType{
@@ -286,6 +288,7 @@ func TestFillDefault(t *testing.T) {
 	}
 
 	expect.TimeZone = y.TimeZone
+	expect.IPFS = y.IPFS
 	expect.Firmware = y.Firmware
 	expect.Firmware.Images = slices.Clone(y.Firmware.Images)
 
@@ -338,6 +341,7 @@ func TestFillDefault(t *testing.T) {
 			ForwardX11Trusted: ptr.Of(false),
 		},
 		TimeZone: ptr.Of("Zulu"),
+		IPFS:     ptr.Of(true),
 		Firmware: Firmware{
 			LegacyBIOS: ptr.Of(true),
 			Images: []FileWithVMType{
@@ -545,6 +549,7 @@ func TestFillDefault(t *testing.T) {
 			ForwardX11Trusted: ptr.Of(false),
 		},
 		TimeZone: ptr.Of("Universal"),
+		IPFS:     ptr.Of(true),
 		Firmware: Firmware{
 			LegacyBIOS: ptr.Of(true),
 		},

pkg/limayaml/limayaml.go

@@ -46,6 +46,7 @@ type LimaYAML struct {
 	Rosetta              Rosetta        `yaml:"rosetta,omitempty" json:"rosetta,omitempty"`
 	Plain                *bool          `yaml:"plain,omitempty" json:"plain,omitempty" jsonschema:"nullable"`
 	TimeZone             *string        `yaml:"timezone,omitempty" json:"timezone,omitempty" jsonschema:"nullable"`
+	IPFS                 *bool          `yaml:"ipfs,omitempty" json:"ipfs,omitempty" jsonschema:"nullable"`
 	NestedVirtualization *bool          `yaml:"nestedVirtualization,omitempty" json:"nestedVirtualization,omitempty" jsonschema:"nullable"`
 }
 
@@ -100,6 +101,7 @@ type File struct {
 	Location string        `yaml:"location" json:"location"` // REQUIRED
 	Arch     Arch          `yaml:"arch,omitempty" json:"arch,omitempty"`
 	Digest   digest.Digest `yaml:"digest,omitempty" json:"digest,omitempty"`
+	Cid      string        `yaml:"cid,omitempty" json:"cid,omitempty"`
 }
 
 type FileWithVMType struct {

pkg/limayaml/validate.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -42,6 +43,18 @@ func validateFileObject(f File, fieldName string) error {
 			return fmt.Errorf("field `%s.digest` is invalid: %s: %w", fieldName, f.Digest.String(), err)
 		}
 	}
+	if f.Cid != "" {
+		if _, err := exec.LookPath("ipfs"); err == nil {
+			cmd := exec.Command("ipfs", "cid", "format", f.Cid)
+			if cid, err := cmd.CombinedOutput(); err != nil || strings.TrimSuffix(string(cid), "\n") != f.Cid {
+				// unfortunately, the `ipfs cid` command does not return any proper exit codes
+				if err == nil {
+					return fmt.Errorf("field `%s.cid` is invalid: %s", fieldName, string(cid))
+				}
+				return fmt.Errorf("field `%s.cid` is invalid: %s: %w", fieldName, f.Cid, err)
+			}
+		}
+	}
 	return nil
 }
 

pkg/qemu/qemu.go

@@ -64,12 +64,12 @@ func EnsureDisk(ctx context.Context, cfg Config) error {
 		var ensuredBaseDisk bool
 		errs := make([]error, len(cfg.LimaYAML.Images))
 		for i, f := range cfg.LimaYAML.Images {
-			if _, err := fileutils.DownloadFile(ctx, baseDisk, f.File, true, "the image", *cfg.LimaYAML.Arch); err != nil {
+			if _, err := fileutils.DownloadFile(ctx, baseDisk, f.File, true, *cfg.LimaYAML.IPFS, "the image", *cfg.LimaYAML.Arch); err != nil {
 				errs[i] = err
 				continue
 			}
 			if f.Kernel != nil {
-				if _, err := fileutils.DownloadFile(ctx, kernel, f.Kernel.File, false, "the kernel", *cfg.LimaYAML.Arch); err != nil {
+				if _, err := fileutils.DownloadFile(ctx, kernel, f.Kernel.File, false, *cfg.LimaYAML.IPFS, "the kernel", *cfg.LimaYAML.Arch); err != nil {
 					errs[i] = err
 					continue
 				}
@@ -81,7 +81,7 @@ func EnsureDisk(ctx context.Context, cfg Config) error {
 				}
 			}
 			if f.Initrd != nil {
-				if _, err := fileutils.DownloadFile(ctx, initrd, *f.Initrd, false, "the initrd", *cfg.LimaYAML.Arch); err != nil {
+				if _, err := fileutils.DownloadFile(ctx, initrd, *f.Initrd, false, *cfg.LimaYAML.IPFS, "the initrd", *cfg.LimaYAML.Arch); err != nil {
 					errs[i] = err
 					continue
 				}
@@ -604,7 +604,7 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er
 				switch f.VMType {
 				case "", limayaml.QEMU:
 					if f.Arch == *y.Arch {
-						if _, err = fileutils.DownloadFile(ctx, downloadedFirmware, f.File, true, "UEFI code "+f.Location, *y.Arch); err != nil {
+						if _, err = fileutils.DownloadFile(ctx, downloadedFirmware, f.File, true, *y.IPFS, "UEFI code "+f.Location, *y.Arch); err != nil {
 							logrus.WithError(err).Warnf("failed to download %q", f.Location)
 							continue loop
 						}

pkg/vz/disk.go

@@ -30,13 +30,13 @@ func EnsureDisk(ctx context.Context, driver *driver.BaseDriver) error {
 		var ensuredBaseDisk bool
 		errs := make([]error, len(driver.Instance.Config.Images))
 		for i, f := range driver.Instance.Config.Images {
-			if _, err := fileutils.DownloadFile(ctx, baseDisk, f.File, true, "the image", *driver.Instance.Config.Arch); err != nil {
+			if _, err := fileutils.DownloadFile(ctx, baseDisk, f.File, true, *driver.Instance.Config.IPFS, "the image", *driver.Instance.Config.Arch); err != nil {
 				errs[i] = err
 				continue
 			}
 			if f.Kernel != nil {
 				// ensure decompress kernel because vz expects it to be decompressed
-				if _, err := fileutils.DownloadFile(ctx, kernel, f.Kernel.File, true, "the kernel", *driver.Instance.Config.Arch); err != nil {
+				if _, err := fileutils.DownloadFile(ctx, kernel, f.Kernel.File, true, *driver.Instance.Config.IPFS, "the kernel", *driver.Instance.Config.Arch); err != nil {
 					errs[i] = err
 					continue
 				}
@@ -48,7 +48,7 @@ func EnsureDisk(ctx context.Context, driver *driver.BaseDriver) error {
 				}
 			}
 			if f.Initrd != nil {
-				if _, err := fileutils.DownloadFile(ctx, initrd, *f.Initrd, false, "the initrd", *driver.Instance.Config.Arch); err != nil {
+				if _, err := fileutils.DownloadFile(ctx, initrd, *f.Initrd, false, *driver.Instance.Config.IPFS, "the initrd", *driver.Instance.Config.Arch); err != nil {
 					errs[i] = err
 					continue
 				}