From 650da2a355e443b9b66027634e3fbf69055b9fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Ma=C5=82ota-W=C3=B3jcik?= <59281144+outofforest@users.noreply.github.com> Date: Tue, 4 Jul 2023 19:19:13 +0200 Subject: [PATCH] Use new mechanism of inflating docker images (#243) --- cmd/main.go | 2 +- commands/build.go | 3 +++ config/build.go | 20 ++++++++++++++++---- functions.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- infra/base/docker.go | 27 ++++++++++++++++++++++----- infra/base/types.go | 2 +- infra/build.go | 26 +++++++++++++------------- 9 files changed, 60 insertions(+), 28 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e0770e9..59c3ccc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -49,7 +49,7 @@ func main() { WithFlavour(executor.NewFlavour(executor.Config{ Router: executor.NewRouter(). RegisterHandler(wire.Execute{}, executor.ExecuteHandler). - RegisterHandler(wire.InitFromDocker{}, executor.NewInitFromDockerHandler()), + RegisterHandler(wire.InflateDockerImage{}, executor.NewInflateDockerImageHandler()), })). Run("osman", func(ctx context.Context, rootCmd *cobra.Command) error { return rootCmd.Execute() diff --git a/commands/build.go b/commands/build.go index 52aa162..f3d8fed 100644 --- a/commands/build.go +++ b/commands/build.go @@ -2,8 +2,10 @@ package commands import ( "fmt" + "os" "github.com/outofforest/ioc/v2" + "github.com/ridge/must" "github.com/spf13/cobra" "github.com/outofforest/osman" @@ -43,5 +45,6 @@ func NewBuildCommand(cmdF *CmdFactory) *cobra.Command { cmd.Flags().StringSliceVar(&buildF.Names, "name", []string{}, "Name of built image, if empty name is derived from corresponding specfile") cmd.Flags().StringSliceVar(&buildF.Tags, "tag", []string{string(description.DefaultTag)}, "Tags assigned to created build") cmd.Flags().BoolVar(&buildF.Rebuild, "rebuild", false, "If set, all parent images are rebuilt even if they exist") + cmd.Flags().StringVar(&buildF.CacheDir, "cache-dir", must.String(os.UserCacheDir())+"/osman", "Path to a directory where files are cached") return cmd } diff --git a/config/build.go b/config/build.go index 2322ba1..4945d25 100644 --- a/config/build.go +++ b/config/build.go @@ -1,31 +1,40 @@ package config import ( + "os" "path/filepath" "strings" + "github.com/ridge/must" + "github.com/outofforest/osman/infra/types" ) -// BuildFactory collects data for build config +// BuildFactory collects data for build config. type BuildFactory struct { - // Names is the list of names for corresponding specfiles + // Names is the list of names for corresponding specfiles. Names []string - // Tags are used to tag the build + // Tags are used to tag the build. Tags []string - // Rebuild forces rebuild of all parent images even if they exist + // Rebuild forces rebuild of all parent images even if they exist. Rebuild bool + + // CacheDir is the directory where cached files are stored. + CacheDir string } // Config creates build config func (f BuildFactory) Config(args Args) Build { + must.OK(os.MkdirAll(f.CacheDir, 0o700)) + config := Build{ SpecFiles: args, Names: f.Names, Tags: make(types.Tags, 0, len(f.Tags)), Rebuild: f.Rebuild, + CacheDir: must.String(filepath.Abs(must.String(filepath.EvalSymlinks(f.CacheDir)))), } for i, specFile := range config.SpecFiles { @@ -52,4 +61,7 @@ type Build struct { // Rebuild forces rebuild of all parent images even if they exist Rebuild bool + + // CacheDir is the directory where cached files are stored. + CacheDir string } diff --git a/functions.go b/functions.go index 0f3d51d..93df004 100644 --- a/functions.go +++ b/functions.go @@ -22,7 +22,7 @@ func Build(ctx context.Context, build config.Build, s storage.Driver, builder *i builds := make([]types.BuildInfo, 0, len(build.SpecFiles)) for i, specFile := range build.SpecFiles { must.OK(os.Chdir(filepath.Dir(specFile))) - buildID, err := builder.BuildFromFile(ctx, specFile, build.Names[i], build.Tags...) + buildID, err := builder.BuildFromFile(ctx, build.CacheDir, specFile, build.Names[i], build.Tags...) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 0541d67..01ca3e1 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/uuid v1.3.0 github.com/outofforest/go-zfs/v3 v3.1.14 github.com/outofforest/ioc/v2 v2.5.2 - github.com/outofforest/isolator v0.7.1 + github.com/outofforest/isolator v0.8.0 github.com/outofforest/logger v0.4.0 github.com/outofforest/parallel v0.2.3 github.com/outofforest/run v0.6.0 diff --git a/go.sum b/go.sum index cb6ffae..40a36e1 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/outofforest/go-zfs/v3 v3.1.14 h1:F0MosonFuGwiIotlMVKoMsvAmISnOWOAncev github.com/outofforest/go-zfs/v3 v3.1.14/go.mod h1:H8SgVKbvhso4bQGQMn6P21mFEd3oDOIaBH2w0Bcsu3c= github.com/outofforest/ioc/v2 v2.5.2 h1:4mNzLuzoZTXL/cO0qf1TrSYvejMgbZz5OUhdLzAUbek= github.com/outofforest/ioc/v2 v2.5.2/go.mod h1:yI+FHuHchC/t6nVo3WJ96qEgCXdHQKFI/4wW2/75YcU= -github.com/outofforest/isolator v0.7.1 h1:S8Gk8H0UB8QpiHhrS+qqVWjCVi2icgiffeYDPymz/k0= -github.com/outofforest/isolator v0.7.1/go.mod h1:02EyC+kX65/xHY0TWHec52JmYLJ8mb+04Cd+Sn2lN5s= +github.com/outofforest/isolator v0.8.0 h1:fBiM2pX2hlinEVpKWOt6+mdbXUooUHyUtyscnzpiQtU= +github.com/outofforest/isolator v0.8.0/go.mod h1:oybGTl8quQf7ra2cq0FwjX6LbgBwZybOErS5oeh2DJc= github.com/outofforest/libexec v0.3.9 h1:KvVLuKDHqpydwNoKm+j36hi9DVPU61X4oHonlZ5cw88= github.com/outofforest/libexec v0.3.9/go.mod h1:J2rUB/m0ER8UNOHd3/UQM55bvh1cbMwhb8gibeF/zyo= github.com/outofforest/logger v0.3.3/go.mod h1:+M5sO17Va9V33t28Qs9VqRQ8bFV501Uhq2PtQY+R3Ms= diff --git a/infra/base/docker.go b/infra/base/docker.go index ef16696..926d271 100644 --- a/infra/base/docker.go +++ b/infra/base/docker.go @@ -2,6 +2,8 @@ package base import ( "context" + "os" + "path/filepath" "github.com/outofforest/isolator" "github.com/outofforest/isolator/wire" @@ -20,18 +22,32 @@ type dockerInitializer struct { } // Init fetches image from docker registry and integrates it inside directory -func (f *dockerInitializer) Init(ctx context.Context, dir string, buildKey types.BuildKey) error { +func (f *dockerInitializer) Init(ctx context.Context, cacheDir, dir string, buildKey types.BuildKey) error { return parallel.Run(ctx, func(ctx context.Context, spawn parallel.SpawnFn) error { incoming := make(chan interface{}) outgoing := make(chan interface{}) + cacheDir := filepath.Join(cacheDir, "docker-images") + if err := os.MkdirAll(cacheDir, 0o700); err != nil { + return errors.WithStack(err) + } + spawn("isolator", parallel.Fail, func(ctx context.Context) error { return isolator.Run(ctx, isolator.Config{ Dir: dir, Types: []interface{}{ wire.Result{}, }, - Executor: wire.Config{NoStandardMounts: true}, + Executor: wire.Config{ + NoStandardMounts: true, + Mounts: []wire.Mount{ + { + Host: cacheDir, + Container: "/.docker-cache", + Writable: true, + }, + }, + }, Incoming: incoming, Outgoing: outgoing, }) @@ -40,9 +56,10 @@ func (f *dockerInitializer) Init(ctx context.Context, dir string, buildKey types select { case <-ctx.Done(): return errors.WithStack(ctx.Err()) - case outgoing <- wire.InitFromDocker{ - Image: buildKey.Name, - Tag: string(buildKey.Tag), + case outgoing <- wire.InflateDockerImage{ + CacheDir: "/.docker-cache", + Image: buildKey.Name, + Tag: string(buildKey.Tag), }: } diff --git a/infra/base/types.go b/infra/base/types.go index 5163199..eab5d16 100644 --- a/infra/base/types.go +++ b/infra/base/types.go @@ -9,5 +9,5 @@ import ( // Initializer initializes base image type Initializer interface { // Init installs base image inside directory - Init(ctx context.Context, dir string, buildKey types.BuildKey) error + Init(ctx context.Context, cacheDir, dir string, buildKey types.BuildKey) error } diff --git a/infra/build.go b/infra/build.go index bf39cc2..188b98e 100644 --- a/infra/build.go +++ b/infra/build.go @@ -44,32 +44,32 @@ type Builder struct { } // BuildFromFile builds image from spec file -func (b *Builder) BuildFromFile(ctx context.Context, specFile, name string, tags ...types.Tag) (types.BuildID, error) { - return b.buildFromFile(ctx, map[types.BuildKey]bool{}, specFile, name, tags...) +func (b *Builder) BuildFromFile(ctx context.Context, cacheDir string, specFile, name string, tags ...types.Tag) (types.BuildID, error) { + return b.buildFromFile(ctx, cacheDir, map[types.BuildKey]bool{}, specFile, name, tags...) } // Build builds images -func (b *Builder) Build(ctx context.Context, img *description.Descriptor) (types.BuildID, error) { - return b.build(ctx, map[types.BuildKey]bool{}, img) +func (b *Builder) Build(ctx context.Context, cacheDir string, img *description.Descriptor) (types.BuildID, error) { + return b.build(ctx, cacheDir, map[types.BuildKey]bool{}, img) } -func (b *Builder) buildFromFile(ctx context.Context, stack map[types.BuildKey]bool, specFile, name string, tags ...types.Tag) (types.BuildID, error) { +func (b *Builder) buildFromFile(ctx context.Context, cacheDir string, stack map[types.BuildKey]bool, specFile, name string, tags ...types.Tag) (types.BuildID, error) { commands, err := b.parser.Parse(specFile) if err != nil { return "", err } - return b.build(ctx, stack, description.Describe(name, tags, commands...)) + return b.build(ctx, cacheDir, stack, description.Describe(name, tags, commands...)) } -func (b *Builder) initialize(ctx context.Context, buildKey types.BuildKey, path string) (retErr error) { +func (b *Builder) initialize(ctx context.Context, cacheDir string, buildKey types.BuildKey, path string) (retErr error) { if buildKey.Name == "scratch" { return nil } // permissions on path dir has to be set to 755 to allow read access for everyone so linux boots correctly - return b.initializer.Init(ctx, path, buildKey) + return b.initializer.Init(ctx, cacheDir, path, buildKey) } -func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img *description.Descriptor) (retBuildID types.BuildID, retErr error) { +func (b *Builder) build(ctx context.Context, cacheDir string, stack map[types.BuildKey]bool, img *description.Descriptor) (retBuildID types.BuildID, retErr error) { if !types.IsNameValid(img.Name()) { return "", errors.Errorf("name %s is invalid", img.Name()) } @@ -130,7 +130,7 @@ func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img return "", err } - if err := b.initialize(ctx, types.NewBuildKey(img.Name(), tags[0]), path); err != nil { + if err := b.initialize(ctx, cacheDir, types.NewBuildKey(img.Name(), tags[0]), path); err != nil { return "", err } } else { @@ -158,7 +158,7 @@ func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img case errors.Is(err, types.ErrImageDoesNotExist): // If image does not exist try to build it from file in the current directory but only if tag is a default one if srcBuildKey.Tag == description.DefaultTag { - _, err = b.buildFromFile(ctx, stack, srcBuildKey.Name, srcBuildKey.Name, description.DefaultTag) + _, err = b.buildFromFile(ctx, cacheDir, stack, srcBuildKey.Name, srcBuildKey.Name, description.DefaultTag) } default: return types.BuildInfo{}, err @@ -169,9 +169,9 @@ func (b *Builder) build(ctx context.Context, stack map[types.BuildKey]bool, img case errors.Is(err, types.ErrImageDoesNotExist): if baseImage := b.repo.Retrieve(srcBuildKey); baseImage != nil { // If spec file does not exist, try building from repository - _, err = b.build(ctx, stack, baseImage) + _, err = b.build(ctx, cacheDir, stack, baseImage) } else { - _, err = b.build(ctx, stack, description.Describe(srcBuildKey.Name, types.Tags{srcBuildKey.Tag})) + _, err = b.build(ctx, cacheDir, stack, description.Describe(srcBuildKey.Name, types.Tags{srcBuildKey.Tag})) } default: return types.BuildInfo{}, err