diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9dbef22c77..742eedc218 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-12-04T15:25:22Z by kres 232fe63. +# Generated on 2024-12-06T17:21:12Z by kres 1ebe796. name: default concurrency: @@ -1606,6 +1606,105 @@ jobs: TF_SCRIPT_DIR: _out/contrib run: | make e2e-cloud-tf + integration-image-cache: + permissions: + actions: read + contents: write + issues: read + packages: write + pull-requests: read + runs-on: + - self-hosted + - talos + if: contains(fromJSON(needs.default.outputs.labels), 'integration/image-cache') + needs: + - default + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@v1.3.0 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: Download artifacts + if: github.event_name != 'schedule' + uses: actions/download-artifact@v4 + with: + name: talos-artifacts + path: _out + - name: Fix artifact permissions + if: github.event_name != 'schedule' + run: | + xargs -a _out/executable-artifacts -I {} chmod +x {} + - name: ci-temp-release-tag + if: github.event_name != 'schedule' + run: | + make ci-temp-release-tag + - name: uki-certs + if: github.event_name == 'schedule' + env: + PLATFORM: linux/amd64 + run: | + make uki-certs + - name: image-cache + env: + IMAGE_REGISTRY: registry.dev.siderolabs.io + MORE_IMAGES: alpine;registry.k8s.io/conformance:v1.32.0-rc.1;registry.k8s.io/e2e-test-images/busybox:1.36.1-1;registry.k8s.io/e2e-test-images/agnhost:2.53;registry.k8s.io/e2e-test-images/httpd:2.4.38-4;registry.k8s.io/e2e-test-images/nonewprivs:1.3;registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7;registry.k8s.io/e2e-test-images/nautilus:1.7;registry.k8s.io/e2e-test-images/sample-apiserver:1.29.2;registry.k8s.io/e2e-test-images/nginx:1.14-4;registry.k8s.io/etcd:3.5.16-0;registry.k8s.io/e2e-test-images/httpd:2.4.39-4 + PLATFORM: linux/amd64,linux/arm64 + PUSH: "true" + run: | + make cache-create + - name: e2e-image-cache + env: + GITHUB_STEP_NAME: ${{ github.job}}-e2e-image-cache + IMAGE_REGISTRY: registry.dev.siderolabs.io + REGISTRY_MIRROR_FLAGS: "no" + SHORT_INTEGRATION_TEST: "yes" + VIA_MAINTENANCE_MODE: "true" + WITH_CONFIG_PATCH: '@hack/test/patches/image-cache.yaml' + WITH_ISO: "true" + run: | + sudo -E make e2e-qemu + - name: save artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: talos-logs-integration-image-cache + path: |- + /tmp/logs-*.tar.gz + /tmp/support-*.zip + retention-days: "5" integration-image-factory: permissions: actions: read diff --git a/.kres.yaml b/.kres.yaml index 9f9308ca4b..d6000eedaf 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -72,6 +72,7 @@ spec: - integration-images - integration-reproducibility-test - integration-cloud-images + - integration-image-cache - integration-image-factory - integration-aws - integration-aws-nvidia-oss @@ -1514,6 +1515,60 @@ spec: PLATFORM: linux/amd64,linux/arm64 IMAGE_REGISTRY: registry.dev.siderolabs.io - name: cloud-images + - name: integration-image-cache + buildxOptions: + enabled: true + depends: + - default + runners: + - self-hosted + - talos + triggerLabels: + - integration/image-cache + steps: + - name: download-artifacts + conditions: + - not-on-schedule + artifactStep: + type: download + artifactName: talos-artifacts + artifactPath: _out + - name: ci-temp-release-tag + conditions: + - not-on-schedule + - name: uki-certs + conditions: + - only-on-schedule + environment: + PLATFORM: linux/amd64 + - name: image-cache + command: cache-create + environment: + PLATFORM: linux/amd64,linux/arm64 + IMAGE_REGISTRY: registry.dev.siderolabs.io + PUSH: true + MORE_IMAGES: "alpine;registry.k8s.io/conformance:v1.32.0-rc.1;registry.k8s.io/e2e-test-images/busybox:1.36.1-1;registry.k8s.io/e2e-test-images/agnhost:2.53;registry.k8s.io/e2e-test-images/httpd:2.4.38-4;registry.k8s.io/e2e-test-images/nonewprivs:1.3;registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7;registry.k8s.io/e2e-test-images/nautilus:1.7;registry.k8s.io/e2e-test-images/sample-apiserver:1.29.2;registry.k8s.io/e2e-test-images/nginx:1.14-4;registry.k8s.io/etcd:3.5.16-0;registry.k8s.io/e2e-test-images/httpd:2.4.39-4" + - name: e2e-image-cache + command: e2e-qemu + withSudo: true + environment: + GITHUB_STEP_NAME: ${{ github.job}}-e2e-image-cache + IMAGE_REGISTRY: registry.dev.siderolabs.io + REGISTRY_MIRROR_FLAGS: no + SHORT_INTEGRATION_TEST: yes + VIA_MAINTENANCE_MODE: true + WITH_CONFIG_PATCH: '@hack/test/patches/image-cache.yaml' + WITH_ISO: true + - name: save-talos-logs + conditions: + - always + artifactStep: + type: upload + artifactName: talos-logs-integration-image-cache + disableExecutableListGeneration: true + artifactPath: /tmp/logs-*.tar.gz + additionalArtifacts: + - "/tmp/support-*.zip" - name: integration-image-factory buildxOptions: enabled: true diff --git a/Makefile b/Makefile index 428e8bf3ef..d441b1fcc5 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,7 @@ SHORT_INTEGRATION_TEST ?= CUSTOM_CNI_URL ?= INSTALLER_ARCH ?= all IMAGER_ARGS ?= +MORE_IMAGES ?= CGO_ENABLED ?= 0 GO_BUILDFLAGS ?= @@ -463,6 +464,12 @@ uki-certs: talosctl ## Generate test certificates for SecureBoot/PCR Signing @$(TALOSCTL_EXECUTABLE) gen secureboot pcr @$(TALOSCTL_EXECUTABLE) gen secureboot database +.PHONY: cache-create +cache-create: installer imager ## Generate image cache. + @( $(TALOSCTL_EXECUTABLE) images default | grep -v 'siderolabs/installer'; echo "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)"; echo "$(MORE_IMAGES)" | tr ';' '\n' ) | $(TALOSCTL_EXECUTABLE) images cache-create --image-cache-path=/tmp/cache.tar --images=- --force + @crane push /tmp/cache.tar $(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG) + @$(MAKE) image-iso IMAGER_ARGS="--image-cache=$(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG) --extra-kernel-arg='console=ttyS0'" + # Code Quality api-descriptors: ## Generates API descriptors used to detect breaking API changes. diff --git a/hack/test/e2e-qemu.sh b/hack/test/e2e-qemu.sh index 51e6c0aef4..e91f362053 100755 --- a/hack/test/e2e-qemu.sh +++ b/hack/test/e2e-qemu.sh @@ -131,6 +131,14 @@ case "${WITH_CONFIG_PATCH:-false}" in ;; esac +case "${WITH_ISO:-false}" in + false) + ;; + *) + QEMU_FLAGS+=("--iso-path=${ARTIFACTS}/metal-amd64.iso") + ;; +esac + case "${WITH_CONFIG_PATCH_WORKER:-false}" in false) ;; diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh index 4a8241dd11..75089755be 100755 --- a/hack/test/e2e.sh +++ b/hack/test/e2e.sh @@ -142,6 +142,12 @@ function dump_cluster_state { } function build_registry_mirrors { + if [[ "${REGISTRY_MIRROR_FLAGS:-yes}" == "no" ]]; then + REGISTRY_MIRROR_FLAGS=() + + return + fi + if [[ "${CI:-false}" == "true" ]]; then REGISTRY_MIRROR_FLAGS=() @@ -151,9 +157,6 @@ function build_registry_mirrors { REGISTRY_MIRROR_FLAGS+=("--registry-mirror=${registry}=http://${addr}:5000") done - else - # use the value from the environment, if present - REGISTRY_MIRROR_FLAGS=("${REGISTRY_MIRROR_FLAGS:-}") fi } diff --git a/hack/test/patches/image-cache.yaml b/hack/test/patches/image-cache.yaml new file mode 100644 index 0000000000..76b1585856 --- /dev/null +++ b/hack/test/patches/image-cache.yaml @@ -0,0 +1,25 @@ +machine: + features: + imageCache: + localEnabled: true + registries: + mirrors: + "*": + skipFallback: true + endpoints: + - http://172.20.0.251:65000 + k8s.gcr.io: + skipFallback: true + endpoints: + - http://172.20.0.251:65000 + registry.k8s.io: + skipFallback: true + endpoints: + - http://172.20.0.251:65000 +--- +apiVersion: v1alpha1 +kind: VolumeConfig +name: IMAGECACHE +provisioning: + diskSelector: + match: 'system_disk' diff --git a/internal/integration/cli/image.go b/internal/integration/cli/image.go index 32c624625c..4f58391422 100644 --- a/internal/integration/cli/image.go +++ b/internal/integration/cli/image.go @@ -47,10 +47,19 @@ func (suite *ImageSuite) TestList() { ) } +var imageCacheQuery = []string{"get", "imagecacheconfig", "--output", "jsonpath='{.spec.copyStatus}'"} + // TestPull verifies pulling images to the CRI. func (suite *ImageSuite) TestPull() { + const image = "registry.k8s.io/kube-apiserver:v1.27.0" + node := suite.RandomDiscoveredNodeInternalIP() - image := "registry.k8s.io/kube-apiserver:v1.27.0" + + if stdout, _ := suite.RunCLI(imageCacheQuery); strings.Contains(stdout, "ready") { + suite.T().Logf("skipping as the image cache is present") + + return + } suite.RunCLI([]string{"image", "pull", "--nodes", node, image}, base.StdoutEmpty(), diff --git a/internal/integration/cli/list.go b/internal/integration/cli/list.go index fd8c3a6468..2999f10598 100644 --- a/internal/integration/cli/list.go +++ b/internal/integration/cli/list.go @@ -7,7 +7,6 @@ package cli import ( - "fmt" "os" "regexp" "strings" @@ -40,10 +39,17 @@ func (suite *ListSuite) TestSuccess() { // TestDepth tests various combinations of --recurse and --depth flags. func (suite *ListSuite) TestDepth() { - suite.T().Parallel() - node := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane) + // Expected maximum number of separators in the output + // In plain terms, it's the maximum depth of the directory tree + maxSeps := 5 + + if stdout, _ := suite.RunCLI(imageCacheQuery); strings.Contains(stdout, "ready") { + // Image cache paths parts are longer + maxSeps = 8 + } + // checks that enough separators are encountered in the output runAndCheck := func(t *testing.T, expectedSeparators int, flags ...string) { args := append([]string{"list", "--nodes", node, "/system"}, flags...) @@ -59,24 +65,28 @@ func (suite *ListSuite) TestDepth() { for _, line := range lines[2:] { actualSeparators := strings.Count(strings.Fields(line)[1], string(os.PathSeparator)) - msg := fmt.Sprintf( - "too many separators (actualSeparators = %d, expectedSeparators = %d)\nflags: %s\nlines:\n%s", - actualSeparators, expectedSeparators, strings.Join(flags, " "), strings.Join(lines, "\n"), - ) - if !assert.LessOrEqual(t, actualSeparators, expectedSeparators, msg) { + if !assert.LessOrEqual( + t, + actualSeparators, + expectedSeparators, + "too many separators, flags: %s\nlines:\n%s", + strings.Join(flags, " "), + stdout, + ) { return } - if maxActualSeparators < actualSeparators { - maxActualSeparators = actualSeparators - } + maxActualSeparators = max(maxActualSeparators, actualSeparators) } - msg := fmt.Sprintf( - "not enough separators (maxActualSeparators = %d, expectedSeparators = %d)\nflags: %s\nlines:\n%s", - maxActualSeparators, expectedSeparators, strings.Join(flags, " "), strings.Join(lines, "\n"), + assert.Equal( + t, + expectedSeparators, + maxActualSeparators, + "not enough separators, \nflags: %s\nlines:\n%s", + strings.Join(flags, " "), + stdout, ) - assert.Equal(t, maxActualSeparators, expectedSeparators, msg) } for _, test := range []struct { @@ -84,19 +94,15 @@ func (suite *ListSuite) TestDepth() { flags []string }{ {separators: 0}, - {separators: 0, flags: []string{"--recurse=false"}}, - {separators: 0, flags: []string{"--depth=-1"}}, {separators: 0, flags: []string{"--depth=0"}}, {separators: 0, flags: []string{"--depth=1"}}, {separators: 1, flags: []string{"--depth=2"}}, {separators: 2, flags: []string{"--depth=3"}}, - - {separators: 5, flags: []string{"--recurse=true"}}, + {separators: maxSeps, flags: []string{"--recurse=true"}}, } { suite.Run(strings.Join(test.flags, ","), func() { - suite.T().Parallel() runAndCheck(suite.T(), test.separators, test.flags...) }) } diff --git a/pkg/imager/cache/cache.go b/pkg/imager/cache/cache.go index df2c12ed22..9f5af3726b 100644 --- a/pkg/imager/cache/cache.go +++ b/pkg/imager/cache/cache.go @@ -37,6 +37,15 @@ const ( manifestsDir = "manifests" ) +// rewriteRegistry name back to workaround https://github.com/google/go-containerregistry/pull/69. +func rewriteRegistry(registryName, origRef string) string { + if registryName == name.DefaultRegistry && !strings.HasPrefix(origRef, name.DefaultRegistry+"/") { + return "docker.io" + } + + return registryName +} + // Generate generates a cache tarball from the given images. // //nolint:gocyclo,cyclop @@ -65,9 +74,7 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa return err } - nameOptions := []name.Option{ - name.StrictValidation, - } + var nameOptions []name.Option craneOpts := []crane.Option{ crane.WithAuthFromKeychain( @@ -99,17 +106,17 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa return fmt.Errorf("parsing reference %q: %w", src, err) } - referenceDir := filepath.Join(tmpDir, manifestsDir, ref.Context().RegistryStr(), ref.Context().RepositoryStr(), "reference") - digestDir := filepath.Join(tmpDir, manifestsDir, ref.Context().RegistryStr(), ref.Context().RepositoryStr(), "digest") + referenceDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), ref.Context().RepositoryStr(), "reference") + digestDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), ref.Context().RepositoryStr(), "digest") - // get the tag from the reference (if it's there) - var tag name.Tag + // if the reference was parsed as a tag, use it + tag, ok := ref.(name.Tag) - base, _, ok := strings.Cut(src, "@") if !ok { - tag, _ = name.NewTag(src, nameOptions...) //nolint:errcheck - } else { - tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck + if base, _, ok := strings.Cut(src, "@"); ok { + // if the reference was a digest, but contained a tag, re-parse it + tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck + } } if err = os.MkdirAll(referenceDir, 0o755); err != nil { @@ -121,11 +128,11 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa } manifest, err := crane.Manifest( - src, + ref.String(), craneOpts..., ) if err != nil { - return fmt.Errorf("fetching manifest %q: %w", src, err) + return fmt.Errorf("fetching manifest %q: %w", ref.String(), err) } rmt, err := remote.Get( @@ -133,7 +140,7 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa remoteOpts..., ) if err != nil { - return fmt.Errorf("fetching image %q: %w", src, err) + return fmt.Errorf("fetching image %q: %w", ref.String(), err) } if tag.TagStr() != "" {