✨ feat: implement initial benchmark tests

.github/workflows/benchmark.yaml

@@ -0,0 +1,118 @@
+---
+name: performance
+
+on:
+  pull_request:
+    types: [opened, reopened]
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Install dependencies
+        run: go mod download
+
+      - name: Set Golang binaries path
+        run: echo "`go env GOPATH`/bin" >> $GITHUB_PATH
+
+      - name: Run golang benchmarks
+        run: go test -run="^$" -bench=. -count=10 ./internal/... > benchmark.txt
+
+      - name: Download baseline benchmark
+        id: download
+        uses: actions/download-artifact@v4
+        with:
+          name: baseline-benchmark
+          path: stats/benchmark.txt
+        continue-on-error: true
+
+      - name: Use current benchmark as reference
+        if: steps.download.outcome != 'success'
+        run: |
+          mkdir stats
+          cp benchmark.txt stats/benchmark.txt
+
+      - name: Install benchstat
+        run: go install golang.org/x/perf/cmd/benchstat@latest
+
+      - name: Compare benchmark results
+        run: |
+          benchstat -table col -row .name -format=csv stats/benchmark.txt \
+          benchmark.txt | grep '^[A-Z]' > stats.csv
+
+      - name: Upload benchstat results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchstat-results
+          path: stats.csv
+          overwrite: true
+
+  parse-results:
+    runs-on: ubuntu-latest
+    needs: [benchmark]
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Download benchstats result
+        uses: actions/download-artifact@v4
+        with:
+          name: benchstat-results
+          path: stats.csv
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pandas numpy tabulate
+
+      - name: Review the results
+        id: results
+        run: |
+          import pandas as pd
+          import os
+
+          # Threshold for increase in compute resource utilization
+          threshold: int = 10
+          df = pd.read_csv('stats.csv',
+            names=['name','reference','secs/op','benchmark','_secs/op','vs_base','P'])
+
+          # Render the benchstat output to markdown
+          out = df.to_markdown()
+
+          s1 = df.vs_base.str.replace('~','0')
+          s1 = pd.to_numeric(s1.str.rstrip('%'))
+          breach = "true" if len(s1[s1 > threshold]) > 0 else "false"
+
+          gh_output = os.environ['GITHUB_OUTPUT']
+          with open(gh_output, 'a') as f:
+            # vs_base captures change in compute resource utilization
+            print(f'breach={breach}', file=f)
+            # capture benchstat results in markdown
+            print(f'report={out}', file=f)
+        shell: python
+
+      - name: Post results
+        uses: actions/github-script@v7
+        script: |
+          github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: ${{ steps.results.output.report }}
+          })
+
+      - name: Parse 'vs base' results
+        run: |
+          if [ "${{ steps.results.output.breach }}" == "true" ]
+          then
+            exit 1
+          else
+            exit 0
+        shell: bash

internal/catalogmetadata/cache/cache_test.go

@@ -342,3 +342,36 @@ func equalFilesystems(expected, actual fs.FS) error {
 	}
 	return errors.Join(cmpErrs...)
 }
+
+func BenchmarkFilesystemCache(b *testing.B) {
+	b.StopTimer()
+	catalog := &catalogd.ClusterCatalog{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-catalog",
+		},
+		Status: catalogd.ClusterCatalogStatus{
+			ResolvedSource: &catalogd.ResolvedCatalogSource{
+				Type: catalogd.SourceTypeImage,
+				Image: &catalogd.ResolvedImageSource{
+					ResolvedRef: "fake/catalog@sha256:fakesha",
+				},
+			},
+		},
+	}
+
+	cacheDir := b.TempDir()
+
+	tripper := &mockTripper{}
+	tripper.content = make(fstest.MapFS)
+	maps.Copy(tripper.content, defaultFS)
+	httpClient := http.DefaultClient
+	httpClient.Transport = tripper
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		c := cache.NewFilesystemCache(cacheDir, func() (*http.Client, error) {
+			return httpClient, nil
+		})
+		_, _ = c.FetchCatalogContents(context.Background(), catalog)
+	}
+}

internal/resolve/catalog_test.go

@@ -589,3 +589,23 @@ func genPackage(pkg string) *declcfg.DeclarativeConfig {
 		Deprecations: []declcfg.Deprecation{packageDeprecation(pkg)},
 	}
 }
+
+func BenchmarkResolve(b *testing.B) {
+	defer featuregatetesting.SetFeatureGateDuringTest(b, features.OperatorControllerFeatureGate, features.ForceSemverUpgradeConstraints, true)()
+	pkgName := randPkg()
+	w := staticCatalogWalker{
+		"a": func() (*declcfg.DeclarativeConfig, error) { return &declcfg.DeclarativeConfig{}, nil },
+		"b": func() (*declcfg.DeclarativeConfig, error) { return &declcfg.DeclarativeConfig{}, nil },
+		"c": func() (*declcfg.DeclarativeConfig, error) { return genPackage(pkgName), nil },
+	}
+	r := CatalogResolver{WalkCatalogsFunc: w.WalkCatalogs}
+	ce := buildFooClusterExtension(pkgName, "", "", ocv1alpha1.UpgradeConstraintPolicyEnforce)
+	installedBundle := &ocv1alpha1.BundleMetadata{
+		Name:    bundleName(pkgName, "1.0.0"),
+		Version: "1.0.0",
+	}
+
+	for i := 0; i < b.N; i++ {
+		_, _, _, _ = r.Resolve(context.Background(), ce, installedBundle)
+	}
+}

internal/rukpak/convert/registryv1_test.go

@@ -1,6 +1,7 @@
 package convert
 
 import (
+	"context"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -13,9 +14,14 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+
+	"github.com/operator-framework/operator-controller/internal/rukpak/source"
 )
 
 func TestRegistryV1Converter(t *testing.T) {
@@ -451,3 +457,36 @@ func containsObject(obj unstructured.Unstructured, result []client.Object) clien
 	}
 	return nil
 }
+
+func BenchmarkRegistryV1ToHelmChart(b *testing.B) {
+	b.StopTimer()
+
+	cacheDir := b.TempDir()
+	mgr, _ := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{})
+	unpacker, _ := source.NewDefaultUnpacker(mgr, "default", cacheDir)
+
+	logger := zap.New(
+		zap.UseFlagOptions(
+			&zap.Options{
+				Development: true,
+			},
+		),
+	)
+
+	log.SetLogger(logger)
+	ctx := log.IntoContext(context.Background(), logger)
+
+	bundleSource := &source.BundleSource{
+		Type: source.SourceTypeImage,
+		Image: &source.ImageSource{
+			Ref: "quay.io/eochieng/litmus-edge-operator-bundle@sha256:104b294fa1f4c2e45aa0eac32437a51de32dce0eff923eced44a0dddcb7f363f",
+		},
+	}
+
+	unpacked, _ := unpacker.Unpack(ctx, bundleSource)
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = RegistryV1ToHelmChart(ctx, unpacked.Bundle, "default", []string{corev1.NamespaceAll})
+	}
+}

internal/rukpak/source/image_registry.go

@@ -84,6 +84,7 @@ func (i *ImageRegistry) Unpack(ctx context.Context, bundle *BundleSource) (*Resu
 		}
 		authChain, err := k8schain.NewInCluster(ctx, chainOpts)
 		if err != nil {
+			l.Error(err, "we encountered an issue getting auth keychain")
 			return nil, fmt.Errorf("error getting auth keychain: %w", err)
 		}
 
@@ -114,17 +115,16 @@ func (i *ImageRegistry) Unpack(ctx context.Context, bundle *BundleSource) (*Resu
 		hexVal := strings.TrimPrefix(digest.DigestStr(), "sha256:")
 		unpackPath := filepath.Join(i.BaseCachePath, bundle.Name, hexVal)
 		if stat, err := os.Stat(unpackPath); err == nil && stat.IsDir() {
-			l.V(1).Info("found image in filesystem cache", "digest", hexVal)
 			return unpackedResult(os.DirFS(unpackPath), bundle, digest.String()), nil
 		}
 	}
 
 	// always fetch the hash
 	imgDesc, err := remote.Head(imgRef, remoteOpts...)
 	if err != nil {
+		l.Error(err, "failed fetching image descriptor")
 		return nil, fmt.Errorf("error fetching image descriptor: %w", err)
 	}
-	l.V(1).Info("resolved image descriptor", "digest", imgDesc.Digest.String())
 
 	unpackPath := filepath.Join(i.BaseCachePath, bundle.Name, imgDesc.Digest.Hex)
 	if _, err = os.Stat(unpackPath); errors.Is(err, os.ErrNotExist) { //nolint: nestif

internal/rukpak/source/image_registry_test.go

@@ -0,0 +1,41 @@
+package source_test
+
+import (
+	"context"
+	"testing"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	"github.com/operator-framework/operator-controller/internal/rukpak/source"
+)
+
+func BenchmarkUnpack(b *testing.B) {
+	b.StopTimer()
+	cacheDir := b.TempDir()
+	mgr, _ := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{})
+	unpacker, _ := source.NewDefaultUnpacker(mgr, "default", cacheDir)
+
+	logger := zap.New(
+		zap.UseFlagOptions(
+			&zap.Options{
+				Development: true,
+			},
+		),
+	)
+
+	ctx := log.IntoContext(context.Background(), logger)
+
+	bundleSource := &source.BundleSource{
+		Type: source.SourceTypeImage,
+		Image: &source.ImageSource{
+			Ref: "quay.io/eochieng/litmus-edge-operator-bundle@sha256:104b294fa1f4c2e45aa0eac32437a51de32dce0eff923eced44a0dddcb7f363f",
+		},
+	}
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = unpacker.Unpack(ctx, bundleSource)
+	}
+}