Use new mechanism of inflating docker images (#243)

cmd/main.go

@@ -49,7 +49,7 @@ func main() {
 		WithFlavour(executor.NewFlavour(executor.Config{
 			Router: executor.NewRouter().
 				RegisterHandler(wire.Execute{}, executor.ExecuteHandler).
-				RegisterHandler(wire.InitFromDocker{}, executor.NewInitFromDockerHandler()),
+				RegisterHandler(wire.InflateDockerImage{}, executor.NewInflateDockerImageHandler()),
 		})).
 		Run("osman", func(ctx context.Context, rootCmd *cobra.Command) error {
 			return rootCmd.Execute()

commands/build.go

@@ -2,8 +2,10 @@ package commands
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/outofforest/ioc/v2"
+	"github.com/ridge/must"
 	"github.com/spf13/cobra"
 
 	"github.com/outofforest/osman"
@@ -43,5 +45,6 @@ func NewBuildCommand(cmdF *CmdFactory) *cobra.Command {
 	cmd.Flags().StringSliceVar(&buildF.Names, "name", []string{}, "Name of built image, if empty name is derived from corresponding specfile")
 	cmd.Flags().StringSliceVar(&buildF.Tags, "tag", []string{string(description.DefaultTag)}, "Tags assigned to created build")
 	cmd.Flags().BoolVar(&buildF.Rebuild, "rebuild", false, "If set, all parent images are rebuilt even if they exist")
+	cmd.Flags().StringVar(&buildF.CacheDir, "cache-dir", must.String(os.UserCacheDir())+"/osman", "Path to a directory where files are cached")
 	return cmd
 }

config/build.go

@@ -1,31 +1,40 @@
 package config
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/ridge/must"
+
 	"github.com/outofforest/osman/infra/types"
 )
 
-// BuildFactory collects data for build config
+// BuildFactory collects data for build config.
 type BuildFactory struct {
-	// Names is the list of names for corresponding specfiles
+	// Names is the list of names for corresponding specfiles.
 	Names []string
 
-	// Tags are used to tag the build
+	// Tags are used to tag the build.
 	Tags []string
 
-	// Rebuild forces rebuild of all parent images even if they exist
+	// Rebuild forces rebuild of all parent images even if they exist.
 	Rebuild bool
+
+	// CacheDir is the directory where cached files are stored.
+	CacheDir string
 }
 
 // Config creates build config
 func (f BuildFactory) Config(args Args) Build {
+	must.OK(os.MkdirAll(f.CacheDir, 0o700))
+
 	config := Build{
 		SpecFiles: args,
 		Names:     f.Names,
 		Tags:      make(types.Tags, 0, len(f.Tags)),
 		Rebuild:   f.Rebuild,
+		CacheDir:  must.String(filepath.Abs(must.String(filepath.EvalSymlinks(f.CacheDir)))),
 	}
 
 	for i, specFile := range config.SpecFiles {
@@ -52,4 +61,7 @@ type Build struct {
 
 	// Rebuild forces rebuild of all parent images even if they exist
 	Rebuild bool
+
+	// CacheDir is the directory where cached files are stored.
+	CacheDir string
 }

functions.go

@@ -22,7 +22,7 @@ func Build(ctx context.Context, build config.Build, s storage.Driver, builder *i
 	builds := make([]types.BuildInfo, 0, len(build.SpecFiles))
 	for i, specFile := range build.SpecFiles {
 		must.OK(os.Chdir(filepath.Dir(specFile)))
-		buildID, err := builder.BuildFromFile(ctx, specFile, build.Names[i], build.Tags...)
+		buildID, err := builder.BuildFromFile(ctx, build.CacheDir, specFile, build.Names[i], build.Tags...)
 		if err != nil {
 			return nil, err
 		}

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/outofforest/go-zfs/v3 v3.1.14
 	github.com/outofforest/ioc/v2 v2.5.2
-	github.com/outofforest/isolator v0.7.1
+	github.com/outofforest/isolator v0.8.0
 	github.com/outofforest/logger v0.4.0
 	github.com/outofforest/parallel v0.2.3
 	github.com/outofforest/run v0.6.0

go.sum

@@ -29,8 +29,8 @@ github.com/outofforest/go-zfs/v3 v3.1.14 h1:F0MosonFuGwiIotlMVKoMsvAmISnOWOAncev
 github.com/outofforest/go-zfs/v3 v3.1.14/go.mod h1:H8SgVKbvhso4bQGQMn6P21mFEd3oDOIaBH2w0Bcsu3c=
 github.com/outofforest/ioc/v2 v2.5.2 h1:4mNzLuzoZTXL/cO0qf1TrSYvejMgbZz5OUhdLzAUbek=
 github.com/outofforest/ioc/v2 v2.5.2/go.mod h1:yI+FHuHchC/t6nVo3WJ96qEgCXdHQKFI/4wW2/75YcU=
-github.com/outofforest/isolator v0.7.1 h1:S8Gk8H0UB8QpiHhrS+qqVWjCVi2icgiffeYDPymz/k0=
-github.com/outofforest/isolator v0.7.1/go.mod h1:02EyC+kX65/xHY0TWHec52JmYLJ8mb+04Cd+Sn2lN5s=
+github.com/outofforest/isolator v0.8.0 h1:fBiM2pX2hlinEVpKWOt6+mdbXUooUHyUtyscnzpiQtU=
+github.com/outofforest/isolator v0.8.0/go.mod h1:oybGTl8quQf7ra2cq0FwjX6LbgBwZybOErS5oeh2DJc=
 github.com/outofforest/libexec v0.3.9 h1:KvVLuKDHqpydwNoKm+j36hi9DVPU61X4oHonlZ5cw88=
 github.com/outofforest/libexec v0.3.9/go.mod h1:J2rUB/m0ER8UNOHd3/UQM55bvh1cbMwhb8gibeF/zyo=
 github.com/outofforest/logger v0.3.3/go.mod h1:+M5sO17Va9V33t28Qs9VqRQ8bFV501Uhq2PtQY+R3Ms=

infra/base/docker.go

@@ -2,6 +2,8 @@ package base
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 
 	"github.com/outofforest/isolator"
 	"github.com/outofforest/isolator/wire"
@@ -20,18 +22,32 @@ type dockerInitializer struct {
 }
 
 // Init fetches image from docker registry and integrates it inside directory
-func (f *dockerInitializer) Init(ctx context.Context, dir string, buildKey types.BuildKey) error {
+func (f *dockerInitializer) Init(ctx context.Context, cacheDir, dir string, buildKey types.BuildKey) error {
 	return parallel.Run(ctx, func(ctx context.Context, spawn parallel.SpawnFn) error {
 		incoming := make(chan interface{})
 		outgoing := make(chan interface{})
 
+		cacheDir := filepath.Join(cacheDir, "docker-images")
+		if err := os.MkdirAll(cacheDir, 0o700); err != nil {
+			return errors.WithStack(err)
+		}
+
 		spawn("isolator", parallel.Fail, func(ctx context.Context) error {
 			return isolator.Run(ctx, isolator.Config{
 				Dir: dir,
 				Types: []interface{}{
 					wire.Result{},
 				},
-				Executor: wire.Config{NoStandardMounts: true},
+				Executor: wire.Config{
+					NoStandardMounts: true,
+					Mounts: []wire.Mount{
+						{
+							Host:      cacheDir,
+							Container: "/.docker-cache",
+							Writable:  true,
+						},
+					},
+				},
 				Incoming: incoming,
 				Outgoing: outgoing,
 			})
@@ -40,9 +56,10 @@ func (f *dockerInitializer) Init(ctx context.Context, dir string, buildKey types
 			select {
 			case <-ctx.Done():
 				return errors.WithStack(ctx.Err())
-			case outgoing <- wire.InitFromDocker{
-				Image: buildKey.Name,
-				Tag:   string(buildKey.Tag),
+			case outgoing <- wire.InflateDockerImage{
+				CacheDir: "/.docker-cache",
+				Image:    buildKey.Name,
+				Tag:      string(buildKey.Tag),
 			}:
 			}
 

infra/base/types.go

@@ -9,5 +9,5 @@ import (
 // Initializer initializes base image
 type Initializer interface {
 	// Init installs base image inside directory
-	Init(ctx context.Context, dir string, buildKey types.BuildKey) error
+	Init(ctx context.Context, cacheDir, dir string, buildKey types.BuildKey) error
 }

infra/build.go

@@ -44,32 +44,32 @@ type Builder struct {
 }
 
 // BuildFromFile builds image from spec file
-func (b *Builder) BuildFromFile(ctx context.Context, specFile, name string, tags ...types.Tag) (types.BuildID, error) {
-	return b.buildFromFile(ctx, map[types.BuildKey]bool{}, specFile, name, tags...)
+func (b *Builder) BuildFromFile(ctx context.Context, cacheDir string, specFile, name string, tags ...types.Tag) (types.BuildID, error) {
+	return b.buildFromFile(ctx, cacheDir, map[types.BuildKey]bool{}, specFile, name, tags...)
 }
 
 // Build builds images
-func (b *Builder) Build(ctx context.Context, img *description.Descriptor) (types.BuildID, error) {
-	return b.build(ctx, map[types.BuildKey]bool{}, img)
+func (b *Builder) Build(ctx context.Context, cacheDir string, img *description.Descriptor) (types.BuildID, error) {
+	return b.build(ctx, cacheDir, map[types.BuildKey]bool{}, img)
 }
 
-func (b *Builder) buildFromFile(ctx context.Context, stack map[types.BuildKey]bool, specFile, name string, tags ...types.Tag) (types.BuildID, error) {
+func (b *Builder) buildFromFile(ctx context.Context, cacheDir string, stack map[types.BuildKey]bool, specFile, name string, tags ...types.Tag) (types.BuildID, error) {
 	commands, err := b.parser.Parse(specFile)
 	if err != nil {
 		return "", err
 	}
-	return b.build(ctx, stack, description.Describe(name, tags, commands...))
+	return b.build(ctx, cacheDir, stack, description.Describe(name, tags, commands...))
 }
 
-func (b *Builder) initialize(ctx context.Context, buildKey types.BuildKey, path string) (retErr error) {
+func (b *Builder) initialize(ctx context.Context, cacheDir string, buildKey types.BuildKey, path string) (retErr error) {
 	if buildKey.Name == "scratch" {
 		return nil
 	}
 	// permissions on path dir has to be set to 755 to allow read access for everyone so linux boots correctly
-	return b.initializer.Init(ctx, path, buildKey)
+	return b.initializer.Init(ctx, cacheDir, path, buildKey)
 }
 
-func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img *description.Descriptor) (retBuildID types.BuildID, retErr error) {
+func (b *Builder) build(ctx context.Context, cacheDir string, stack map[types.BuildKey]bool, img *description.Descriptor) (retBuildID types.BuildID, retErr error) {
 	if !types.IsNameValid(img.Name()) {
 		return "", errors.Errorf("name %s is invalid", img.Name())
 	}
@@ -130,7 +130,7 @@ func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img
 			return "", err
 		}
 
-		if err := b.initialize(ctx, types.NewBuildKey(img.Name(), tags[0]), path); err != nil {
+		if err := b.initialize(ctx, cacheDir, types.NewBuildKey(img.Name(), tags[0]), path); err != nil {
 			return "", err
 		}
 	} else {
@@ -158,7 +158,7 @@ func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img
 				case errors.Is(err, types.ErrImageDoesNotExist):
 					// If image does not exist try to build it from file in the current directory but only if tag is a default one
 					if srcBuildKey.Tag == description.DefaultTag {
-						_, err = b.buildFromFile(ctx, stack, srcBuildKey.Name, srcBuildKey.Name, description.DefaultTag)
+						_, err = b.buildFromFile(ctx, cacheDir, stack, srcBuildKey.Name, srcBuildKey.Name, description.DefaultTag)
 					}
 				default:
 					return types.BuildInfo{}, err
@@ -169,9 +169,9 @@ func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img
 				case errors.Is(err, types.ErrImageDoesNotExist):
 					if baseImage := b.repo.Retrieve(srcBuildKey); baseImage != nil {
 						// If spec file does not exist, try building from repository
-						_, err = b.build(ctx, stack, baseImage)
+						_, err = b.build(ctx, cacheDir, stack, baseImage)
 					} else {
-						_, err = b.build(ctx, stack, description.Describe(srcBuildKey.Name, types.Tags{srcBuildKey.Tag}))
+						_, err = b.build(ctx, cacheDir, stack, description.Describe(srcBuildKey.Name, types.Tags{srcBuildKey.Tag}))
 					}
 				default:
 					return types.BuildInfo{}, err