From 965e099f9641d79e416165fb6881f65929d0d11a Mon Sep 17 00:00:00 2001 From: Christoph Ostarek Date: Fri, 11 Oct 2024 11:54:43 +0200 Subject: [PATCH] tools: introduce dockerfile-from-checker this tool checks for inconsistencies in `FROM` in Dockerfiles and reports them Signed-off-by: Christoph Ostarek --- Makefile | 8 + tools/dockerfile-from-checker/.gitignore | 1 + tools/dockerfile-from-checker/Makefile | 6 + tools/dockerfile-from-checker/go.mod | 26 +++ tools/dockerfile-from-checker/go.sum | 76 +++++++++ tools/dockerfile-from-checker/main.go | 194 +++++++++++++++++++++++ 6 files changed, 311 insertions(+) create mode 100644 tools/dockerfile-from-checker/.gitignore create mode 100644 tools/dockerfile-from-checker/Makefile create mode 100644 tools/dockerfile-from-checker/go.mod create mode 100644 tools/dockerfile-from-checker/go.sum create mode 100644 tools/dockerfile-from-checker/main.go diff --git a/Makefile b/Makefile index 2075151f18..e3f9ed9886 100644 --- a/Makefile +++ b/Makefile @@ -432,6 +432,14 @@ test: $(LINUXKIT) test-images-patches | $(DIST) cp pkg/pillar/results.json $(DIST)/ cp pkg/pillar/results.xml $(DIST)/ make -C eve-tools/bpftrace-compiler test + @echo Checking Dockerfiles for inconsistencies + make -C tools/dockerfile-from-checker/ + ./tools/dockerfile-from-checker/dockerfile-from-checker ./ \ + -i ./eve-tools/bpftrace-compiler/Dockerfile \ + -i ./pkg/optee-os/Dockerfile \ + -i pkg/bsp-imx/Dockerfile \ + -i pkg/vtpm/Dockerfile \ + -i pkg/installer/Dockerfile $(QUIET): $@: Succeeded test-profiling: diff --git a/tools/dockerfile-from-checker/.gitignore b/tools/dockerfile-from-checker/.gitignore new file mode 100644 index 0000000000..ce64dbb80c --- /dev/null +++ b/tools/dockerfile-from-checker/.gitignore @@ -0,0 +1 @@ +dockerfile-from-checker diff --git a/tools/dockerfile-from-checker/Makefile b/tools/dockerfile-from-checker/Makefile new file mode 100644 index 0000000000..9dcbd8a49b --- /dev/null +++ b/tools/dockerfile-from-checker/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2024 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +dockerfile-from-checker: go.mod go.sum main.go + go build -o $@ . + diff --git a/tools/dockerfile-from-checker/go.mod b/tools/dockerfile-from-checker/go.mod new file mode 100644 index 0000000000..5fda2f412d --- /dev/null +++ b/tools/dockerfile-from-checker/go.mod @@ -0,0 +1,26 @@ +module dockerfile-checker + +go 1.21 + +toolchain go1.21.5 + +require ( + github.com/moby/buildkit v0.13.0-beta3 + github.com/spf13/cobra v1.6.1 +) + +require ( + github.com/agext/levenshtein v1.2.3 // indirect + github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/docker/docker v25.0.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gotest.tools/v3 v3.5.1 // indirect +) diff --git a/tools/dockerfile-from-checker/go.sum b/tools/dockerfile-from-checker/go.sum new file mode 100644 index 0000000000..43f8fc7913 --- /dev/null +++ b/tools/dockerfile-from-checker/go.sum @@ -0,0 +1,76 @@ +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= +github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/docker v25.0.1+incompatible h1:k5TYd5rIVQRSqcTwCID+cyVA0yRg86+Pcrz1ls0/frA= +github.com/docker/docker v25.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/moby/buildkit v0.13.0-beta3 h1:eefOGE6SsWYHFfymc09Q7VU5i3L9vUs8ZCZVCDXWNOo= +github.com/moby/buildkit v0.13.0-beta3/go.mod h1:tSWWhq1EDM0eB3ngMNDiH2hOOW9fXTyn2uXuOraCLlE= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= +github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/tools/dockerfile-from-checker/main.go b/tools/dockerfile-from-checker/main.go new file mode 100644 index 0000000000..5e9178cc76 --- /dev/null +++ b/tools/dockerfile-from-checker/main.go @@ -0,0 +1,194 @@ +// Copyright (c) 2024 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package main is the only package of this tool +package main + +import ( + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/frontend/dockerfile/parser" + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{} + +func init() { + rootCmd = &cobra.Command{ + Use: "dockerfile-checker ", + Args: cobra.ExactArgs(1), + Run: rootFunc, + } +} + +func rootFunc(cmd *cobra.Command, args []string) { + var paths []string + ignorePaths := make(map[string]struct{}) + + ignoreRelPaths, err := rootCmd.Flags().GetStringSlice("ignore-files") + if err != nil { + log.Fatal(err) + } + for _, ign := range ignoreRelPaths { + absPath, err := filepath.Abs(ign) + if err != nil { + log.Fatal(err) + } + + ignorePaths[absPath] = struct{}{} + } + + err = filepath.Walk(args[0], func(p string, info fs.FileInfo, err error) error { + if info.Name() == "vendor" { + return filepath.SkipDir + } + if info.IsDir() { + return nil + } + + if info.Name() == "Dockerfile" { + absPath, err := filepath.Abs(p) + if err != nil { + log.Fatal(err) + } + _, ok := ignorePaths[absPath] + if !ok { + // path is not on ignore list + paths = append(paths, absPath) + } + } + return nil + }) + + if err != nil { + panic(err) + } + + checkDockerfiles(paths) +} + +func checkDockerfiles(paths []string) { + var f *os.File + var err error + + froms2dockerfile := make(map[string][]string) + for _, filename := range paths { + f, err = os.Open(filename) + if err != nil { + log.Fatalf("could not open %s: %+v", filename, err) + } + defer f.Close() + + dockerfileFroms := parseDockerfile(f) + for _, from := range dockerfileFroms { + fns := append(froms2dockerfile[from], filename) + froms2dockerfile[from] = fns + } + } + + checkInconsistencies(froms2dockerfile) +} + +func main() { + rootCmd.Flags().StringSliceP("ignore-files", "i", []string{}, "ignores specified Dockerfile; can be used several times") + + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func checkInconsistencies(froms2dockerfile map[string][]string) { + type tagFile struct { + tag string + file string + fullname string + } + + image2TagFile := make(map[string][]tagFile) + for from, files := range froms2dockerfile { + for _, file := range files { + splits := strings.Split(from, ":") + if len(splits) < 1 { + continue + } + tag := splits[len(splits)-1] + image := strings.Join(splits[:len(splits)-1], "") + if image == "" { + continue + } + tf := tagFile{ + tag: tag, + file: file, + fullname: from, + } + image2TagFile[image] = append(image2TagFile[image], tf) + } + } + + for _, tfs := range image2TagFile { + for i := 1; i < len(tfs); i++ { + tf := tfs[i] + if tf.tag != tfs[i-1].tag { + fmt.Printf("tags differ for image %s in files %s and %s\n", tf.fullname, tf.file, tfs[i-1].file) + os.Exit(1) + } + } + } +} + +func parseDockerfile(f *os.File) []string { + var dockerfileFroms []string + result, err := parser.Parse(f) + if err != nil { + log.Fatalf("parsing %s failed: %+v", f.Name(), err) + } + + vars := parseVars(result) + var next *parser.Node + for _, node := range result.AST.Children { + if node.Value == "FROM" { + next = node.Next + if next == nil { + break + } + from := expandVariables(next, vars) + dockerfileFroms = append(dockerfileFroms, from) + } + } + + return dockerfileFroms +} + +func expandVariables(next *parser.Node, vars map[string]string) string { + from := next.Value + for key, val := range vars { + from = strings.ReplaceAll(from, fmt.Sprintf("${%s}", key), val) + } + return from +} + +func parseVars(result *parser.Result) map[string]string { + vars := make(map[string]string) + _, metaArgs, err := instructions.Parse(result.AST) + if err != nil { + log.Fatal(err) + } + + for _, argCmd := range metaArgs { + if argCmd.Name() != "ARG" { + continue + } + for _, argCmdArg := range argCmd.Args { + vars[argCmdArg.Key] = argCmdArg.ValueString() + } + } + + return vars +}