diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 000000000..5d7922f3a --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,118 @@ +--- +name: performance + +on: + pull_request: + types: [opened, reopened] + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install dependencies + run: go mod download + + - name: Set Golang binaries path + run: echo "`go env GOPATH`/bin" >> $GITHUB_PATH + + - name: Run golang benchmarks + run: go test -run="^$" -bench=. -count=10 ./internal/... > benchmark.txt + + - name: Download baseline benchmark + id: download + uses: actions/download-artifact@v4 + with: + name: baseline-benchmark + path: stats/benchmark.txt + continue-on-error: true + + - name: Use current benchmark as reference + if: steps.download.outcome != 'success' + run: | + mkdir stats + cp benchmark.txt stats/benchmark.txt + + - name: Install benchstat + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Compare benchmark results + run: | + benchstat -table col -row .name -format=csv stats/benchmark.txt \ + benchmark.txt | grep '^[A-Z]' > stats.csv + + - name: Upload benchstat results + uses: actions/upload-artifact@v4 + with: + name: benchstat-results + path: stats.csv + overwrite: true + + parse-results: + runs-on: ubuntu-latest + needs: [benchmark] + steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download benchstats result + uses: actions/download-artifact@v4 + with: + name: benchstat-results + path: stats.csv + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install pandas numpy tabulate + + - name: Review the results + id: results + run: | + import pandas as pd + import os + + # Threshold for increase in compute resource utilization + threshold: int = 10 + df = pd.read_csv('stats.csv', + names=['name','reference','secs/op','benchmark','_secs/op','vs_base','P']) + + # Render the benchstat output to markdown + out = df.to_markdown() + + s1 = df.vs_base.str.replace('~','0') + s1 = pd.to_numeric(s1.str.rstrip('%')) + breach = "true" if len(s1[s1 > threshold]) > 0 else "false" + + gh_output = os.environ['GITHUB_OUTPUT'] + with open(gh_output, 'a') as f: + # vs_base captures change in compute resource utilization + print(f'breach={breach}', file=f) + # capture benchstat results in markdown + print(f'report={out}', file=f) + shell: python + + - name: Post results + uses: actions/github-script@v7 + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: ${{ steps.results.output.report }} + }) + + - name: Parse 'vs base' results + run: | + if [ "${{ steps.results.output.breach }}" == "true" ] + then + exit 1 + else + exit 0 + shell: bash diff --git a/internal/catalogmetadata/cache/cache_test.go b/internal/catalogmetadata/cache/cache_test.go index 05ea28ec5..a640138f9 100644 --- a/internal/catalogmetadata/cache/cache_test.go +++ b/internal/catalogmetadata/cache/cache_test.go @@ -342,3 +342,36 @@ func equalFilesystems(expected, actual fs.FS) error { } return errors.Join(cmpErrs...) } + +func BenchmarkFilesystemCache(b *testing.B) { + b.StopTimer() + catalog := &catalogd.ClusterCatalog{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-catalog", + }, + Status: catalogd.ClusterCatalogStatus{ + ResolvedSource: &catalogd.ResolvedCatalogSource{ + Type: catalogd.SourceTypeImage, + Image: &catalogd.ResolvedImageSource{ + ResolvedRef: "fake/catalog@sha256:fakesha", + }, + }, + }, + } + + cacheDir := b.TempDir() + + tripper := &mockTripper{} + tripper.content = make(fstest.MapFS) + maps.Copy(tripper.content, defaultFS) + httpClient := http.DefaultClient + httpClient.Transport = tripper + b.StartTimer() + + for i := 0; i < b.N; i++ { + c := cache.NewFilesystemCache(cacheDir, func() (*http.Client, error) { + return httpClient, nil + }) + _, _ = c.FetchCatalogContents(context.Background(), catalog) + } +} diff --git a/internal/resolve/catalog_test.go b/internal/resolve/catalog_test.go index a067bbe79..ead055acf 100644 --- a/internal/resolve/catalog_test.go +++ b/internal/resolve/catalog_test.go @@ -589,3 +589,23 @@ func genPackage(pkg string) *declcfg.DeclarativeConfig { Deprecations: []declcfg.Deprecation{packageDeprecation(pkg)}, } } + +func BenchmarkResolve(b *testing.B) { + defer featuregatetesting.SetFeatureGateDuringTest(b, features.OperatorControllerFeatureGate, features.ForceSemverUpgradeConstraints, true)() + pkgName := randPkg() + w := staticCatalogWalker{ + "a": func() (*declcfg.DeclarativeConfig, error) { return &declcfg.DeclarativeConfig{}, nil }, + "b": func() (*declcfg.DeclarativeConfig, error) { return &declcfg.DeclarativeConfig{}, nil }, + "c": func() (*declcfg.DeclarativeConfig, error) { return genPackage(pkgName), nil }, + } + r := CatalogResolver{WalkCatalogsFunc: w.WalkCatalogs} + ce := buildFooClusterExtension(pkgName, "", "", ocv1alpha1.UpgradeConstraintPolicyEnforce) + installedBundle := &ocv1alpha1.BundleMetadata{ + Name: bundleName(pkgName, "1.0.0"), + Version: "1.0.0", + } + + for i := 0; i < b.N; i++ { + _, _, _, _ = r.Resolve(context.Background(), ce, installedBundle) + } +} diff --git a/internal/rukpak/convert/registryv1_test.go b/internal/rukpak/convert/registryv1_test.go index 33ed9781f..9592ffb67 100644 --- a/internal/rukpak/convert/registryv1_test.go +++ b/internal/rukpak/convert/registryv1_test.go @@ -1,6 +1,7 @@ package convert import ( + "context" "testing" . "github.com/onsi/ginkgo/v2" @@ -13,9 +14,14 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/operator-framework/api/pkg/operators/v1alpha1" + + "github.com/operator-framework/operator-controller/internal/rukpak/source" ) func TestRegistryV1Converter(t *testing.T) { @@ -451,3 +457,36 @@ func containsObject(obj unstructured.Unstructured, result []client.Object) clien } return nil } + +func BenchmarkRegistryV1ToHelmChart(b *testing.B) { + b.StopTimer() + + cacheDir := b.TempDir() + mgr, _ := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + unpacker, _ := source.NewDefaultUnpacker(mgr, "default", cacheDir) + + logger := zap.New( + zap.UseFlagOptions( + &zap.Options{ + Development: true, + }, + ), + ) + + log.SetLogger(logger) + ctx := log.IntoContext(context.Background(), logger) + + bundleSource := &source.BundleSource{ + Type: source.SourceTypeImage, + Image: &source.ImageSource{ + Ref: "quay.io/eochieng/litmus-edge-operator-bundle@sha256:104b294fa1f4c2e45aa0eac32437a51de32dce0eff923eced44a0dddcb7f363f", + }, + } + + unpacked, _ := unpacker.Unpack(ctx, bundleSource) + b.StartTimer() + + for i := 0; i < b.N; i++ { + _, _ = RegistryV1ToHelmChart(ctx, unpacked.Bundle, "default", []string{corev1.NamespaceAll}) + } +} diff --git a/internal/rukpak/source/image_registry.go b/internal/rukpak/source/image_registry.go index a6d6640d4..70ff88dda 100644 --- a/internal/rukpak/source/image_registry.go +++ b/internal/rukpak/source/image_registry.go @@ -84,6 +84,7 @@ func (i *ImageRegistry) Unpack(ctx context.Context, bundle *BundleSource) (*Resu } authChain, err := k8schain.NewInCluster(ctx, chainOpts) if err != nil { + l.Error(err, "we encountered an issue getting auth keychain") return nil, fmt.Errorf("error getting auth keychain: %w", err) } @@ -114,7 +115,6 @@ func (i *ImageRegistry) Unpack(ctx context.Context, bundle *BundleSource) (*Resu hexVal := strings.TrimPrefix(digest.DigestStr(), "sha256:") unpackPath := filepath.Join(i.BaseCachePath, bundle.Name, hexVal) if stat, err := os.Stat(unpackPath); err == nil && stat.IsDir() { - l.V(1).Info("found image in filesystem cache", "digest", hexVal) return unpackedResult(os.DirFS(unpackPath), bundle, digest.String()), nil } } @@ -122,9 +122,9 @@ func (i *ImageRegistry) Unpack(ctx context.Context, bundle *BundleSource) (*Resu // always fetch the hash imgDesc, err := remote.Head(imgRef, remoteOpts...) if err != nil { + l.Error(err, "failed fetching image descriptor") return nil, fmt.Errorf("error fetching image descriptor: %w", err) } - l.V(1).Info("resolved image descriptor", "digest", imgDesc.Digest.String()) unpackPath := filepath.Join(i.BaseCachePath, bundle.Name, imgDesc.Digest.Hex) if _, err = os.Stat(unpackPath); errors.Is(err, os.ErrNotExist) { //nolint: nestif diff --git a/internal/rukpak/source/image_registry_test.go b/internal/rukpak/source/image_registry_test.go new file mode 100644 index 000000000..3b32ad3e0 --- /dev/null +++ b/internal/rukpak/source/image_registry_test.go @@ -0,0 +1,41 @@ +package source_test + +import ( + "context" + "testing" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/operator-framework/operator-controller/internal/rukpak/source" +) + +func BenchmarkUnpack(b *testing.B) { + b.StopTimer() + cacheDir := b.TempDir() + mgr, _ := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + unpacker, _ := source.NewDefaultUnpacker(mgr, "default", cacheDir) + + logger := zap.New( + zap.UseFlagOptions( + &zap.Options{ + Development: true, + }, + ), + ) + + ctx := log.IntoContext(context.Background(), logger) + + bundleSource := &source.BundleSource{ + Type: source.SourceTypeImage, + Image: &source.ImageSource{ + Ref: "quay.io/eochieng/litmus-edge-operator-bundle@sha256:104b294fa1f4c2e45aa0eac32437a51de32dce0eff923eced44a0dddcb7f363f", + }, + } + b.StartTimer() + + for i := 0; i < b.N; i++ { + _, _ = unpacker.Unpack(ctx, bundleSource) + } +}