From c46f95227bd5129ed113907c79123039e727c49c Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Mon, 15 Jul 2024 22:02:11 -0400 Subject: [PATCH 1/5] [v2] Add e2e test with memory store Signed-off-by: Yuri Shkuro --- .github/workflows/ci-e2e-memory.yaml | 44 +++++++++++++++++++ .../internal/integration/e2e_integration.go | 5 ++- .../{es_test.go => elasticsearch_test.go} | 2 +- .../internal/integration/memory_test.go | 27 ++++++++++++ .../{os_test.go => opensearch_test.go} | 2 +- .../integration/storagecleaner/extension.go | 16 ++++--- plugin/storage/memory/factory.go | 9 ++++ plugin/storage/memory/memory.go | 7 +++ 8 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/ci-e2e-memory.yaml rename cmd/jaeger/internal/integration/{es_test.go => elasticsearch_test.go} (93%) create mode 100644 cmd/jaeger/internal/integration/memory_test.go rename cmd/jaeger/internal/integration/{os_test.go => opensearch_test.go} (93%) diff --git a/.github/workflows/ci-e2e-memory.yaml b/.github/workflows/ci-e2e-memory.yaml new file mode 100644 index 00000000000..c33a8058fc5 --- /dev/null +++ b/.github/workflows/ci-e2e-memory.yaml @@ -0,0 +1,44 @@ +name: CIT Badger + +on: + push: + branches: [main] + + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }} + cancel-in-progress: true + +# See https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions +permissions: # added using https://github.com/step-security/secure-workflows + contents: read + +jobs: + badger: + runs-on: ubuntu-latest + strategy: + matrix: + version: [v1, v2] + steps: + - name: Harden Runner + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.x + + - name: Run Memory storage integration tests + run: | + STORAGE=memory make jaeger-v2-storage-integration-test + + - name: Upload coverage to codecov + uses: ./.github/actions/upload-codecov + with: + files: cover.out + flags: badger_${{ matrix.version }} diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 6ddcb538cff..670b7b576d4 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -177,6 +177,7 @@ func createStorageCleanerConfig(t *testing.T, configFile string, storage string) func purge(t *testing.T) { addr := fmt.Sprintf("http://0.0.0.0:%s%s", storagecleaner.Port, storagecleaner.URL) + t.Logf("Purging storage via %s", addr) r, err := http.NewRequestWithContext(context.Background(), http.MethodPost, addr, nil) require.NoError(t, err) @@ -185,6 +186,8 @@ func purge(t *testing.T) { resp, err := client.Do(r) require.NoError(t, err) defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) - require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", string(body)) } diff --git a/cmd/jaeger/internal/integration/es_test.go b/cmd/jaeger/internal/integration/elasticsearch_test.go similarity index 93% rename from cmd/jaeger/internal/integration/es_test.go rename to cmd/jaeger/internal/integration/elasticsearch_test.go index 92b9f0c79e0..2c78bfdb7af 100644 --- a/cmd/jaeger/internal/integration/es_test.go +++ b/cmd/jaeger/internal/integration/elasticsearch_test.go @@ -9,7 +9,7 @@ import ( "github.com/jaegertracing/jaeger/plugin/storage/integration" ) -func TestESStorage(t *testing.T) { +func TestElasticsearchStorage(t *testing.T) { integration.SkipUnlessEnv(t, "elasticsearch") s := &E2EStorageIntegration{ diff --git a/cmd/jaeger/internal/integration/memory_test.go b/cmd/jaeger/internal/integration/memory_test.go new file mode 100644 index 00000000000..9142f2d1080 --- /dev/null +++ b/cmd/jaeger/internal/integration/memory_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "testing" + + "github.com/jaegertracing/jaeger/plugin/storage/integration" +) + +func TestMemoryStorage(t *testing.T) { + integration.SkipUnlessEnv(t, "memory") + + s := &E2EStorageIntegration{ + ConfigFile: "../../config.yaml", + StorageIntegration: integration.StorageIntegration{ + SkipArchiveTest: true, + CleanUp: purge, + }, + } + s.e2eInitialize(t, "memory") + t.Cleanup(func() { + s.e2eCleanUp(t) + }) + s.RunAll(t) +} diff --git a/cmd/jaeger/internal/integration/os_test.go b/cmd/jaeger/internal/integration/opensearch_test.go similarity index 93% rename from cmd/jaeger/internal/integration/os_test.go rename to cmd/jaeger/internal/integration/opensearch_test.go index a5d3e945214..909c0be6510 100644 --- a/cmd/jaeger/internal/integration/os_test.go +++ b/cmd/jaeger/internal/integration/opensearch_test.go @@ -9,7 +9,7 @@ import ( "github.com/jaegertracing/jaeger/plugin/storage/integration" ) -func TestOSStorage(t *testing.T) { +func TestOpenSearchStorage(t *testing.T) { integration.SkipUnlessEnv(t, "opensearch") s := &E2EStorageIntegration{ ConfigFile: "../../config-opensearch.yaml", diff --git a/cmd/jaeger/internal/integration/storagecleaner/extension.go b/cmd/jaeger/internal/integration/storagecleaner/extension.go index 2a8b37016e6..c49d9b7bff0 100644 --- a/cmd/jaeger/internal/integration/storagecleaner/extension.go +++ b/cmd/jaeger/internal/integration/storagecleaner/extension.go @@ -13,6 +13,7 @@ import ( "github.com/gorilla/mux" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/extension" + "go.uber.org/zap" "github.com/jaegertracing/jaeger/cmd/jaeger/internal/extension/jaegerstorage" "github.com/jaegertracing/jaeger/storage" @@ -29,15 +30,15 @@ const ( ) type storageCleaner struct { - config *Config - server *http.Server - settings component.TelemetrySettings + config *Config + server *http.Server + telset component.TelemetrySettings } -func newStorageCleaner(config *Config, telemetrySettings component.TelemetrySettings) *storageCleaner { +func newStorageCleaner(config *Config, telset component.TelemetrySettings) *storageCleaner { return &storageCleaner{ - config: config, - settings: telemetrySettings, + config: config, + telset: telset, } } @@ -74,10 +75,11 @@ func (c *storageCleaner) Start(_ context.Context, host component.Host) error { Handler: r, ReadHeaderTimeout: 3 * time.Second, } + c.telset.Logger.Info("Starting storage cleaner server", zap.String("addr", c.server.Addr)) go func() { if err := c.server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { err = fmt.Errorf("error starting cleaner server: %w", err) - c.settings.ReportStatus(component.NewFatalErrorEvent(err)) + c.telset.ReportStatus(component.NewFatalErrorEvent(err)) } }() diff --git a/plugin/storage/memory/factory.go b/plugin/storage/memory/factory.go index 6c677b03d95..d9bfd6c113a 100644 --- a/plugin/storage/memory/factory.go +++ b/plugin/storage/memory/factory.go @@ -16,6 +16,7 @@ package memory import ( + "context" "flag" "github.com/spf13/viper" @@ -36,6 +37,7 @@ var ( // interface comformance checks _ storage.ArchiveFactory = (*Factory)(nil) _ storage.SamplingStoreFactory = (*Factory)(nil) _ plugin.Configurable = (*Factory)(nil) + _ storage.Purger = (*Factory)(nil) ) // Factory implements storage.Factory and creates storage components backed by memory store. @@ -126,3 +128,10 @@ func (*Factory) CreateLock() (distributedlock.Lock, error) { func (f *Factory) publishOpts() { safeexpvar.SetInt("jaeger_storage_memory_max_traces", int64(f.options.Configuration.MaxTraces)) } + +// Purge removes all data from the Factory's underlying Memory store. +// This function is intended for testing purposes only and should not be used in production environments. +func (f *Factory) Purge(ctx context.Context) error { + f.store.purge(ctx) + return nil +} diff --git a/plugin/storage/memory/memory.go b/plugin/storage/memory/memory.go index f92ae9949e9..9bac7d2b2bd 100644 --- a/plugin/storage/memory/memory.go +++ b/plugin/storage/memory/memory.go @@ -337,3 +337,10 @@ func flattenTags(span *model.Span) model.KeyValues { } return retMe } + +// purge supports Purger interface. +func (st *Store) purge(context.Context) { + st.Lock() + st.perTenant = make(map[string]*Tenant) + st.Unlock() +} From c29429ccae0659ab9755bbc4cc93f55f1c81f124 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Mon, 15 Jul 2024 22:15:54 -0400 Subject: [PATCH 2/5] fix Signed-off-by: Yuri Shkuro --- .github/workflows/ci-e2e-memory.yaml | 2 +- cmd/jaeger/internal/integration/memory_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-e2e-memory.yaml b/.github/workflows/ci-e2e-memory.yaml index c33a8058fc5..0726b8cc276 100644 --- a/.github/workflows/ci-e2e-memory.yaml +++ b/.github/workflows/ci-e2e-memory.yaml @@ -35,7 +35,7 @@ jobs: - name: Run Memory storage integration tests run: | - STORAGE=memory make jaeger-v2-storage-integration-test + STORAGE=memory_v2 make jaeger-v2-storage-integration-test - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov diff --git a/cmd/jaeger/internal/integration/memory_test.go b/cmd/jaeger/internal/integration/memory_test.go index 9142f2d1080..a2592b04a38 100644 --- a/cmd/jaeger/internal/integration/memory_test.go +++ b/cmd/jaeger/internal/integration/memory_test.go @@ -10,7 +10,7 @@ import ( ) func TestMemoryStorage(t *testing.T) { - integration.SkipUnlessEnv(t, "memory") + integration.SkipUnlessEnv(t, "memory_v2") s := &E2EStorageIntegration{ ConfigFile: "../../config.yaml", From 043ea77fd2f4e86246741ed9742e6788f3927a22 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Mon, 15 Jul 2024 22:29:52 -0400 Subject: [PATCH 3/5] fix Signed-off-by: Yuri Shkuro --- .../integration/storagecleaner/extension_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/jaeger/internal/integration/storagecleaner/extension_test.go b/cmd/jaeger/internal/integration/storagecleaner/extension_test.go index a150e0f00ac..6f1290cf0b9 100644 --- a/cmd/jaeger/internal/integration/storagecleaner/extension_test.go +++ b/cmd/jaeger/internal/integration/storagecleaner/extension_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" + "go.uber.org/zap/zaptest" "github.com/jaegertracing/jaeger/cmd/jaeger/internal/extension/jaegerstorage" "github.com/jaegertracing/jaeger/storage" @@ -84,7 +85,9 @@ func TestStorageCleanerExtension(t *testing.T) { TraceStorage: "storage", Port: Port, } - s := newStorageCleaner(config, component.TelemetrySettings{}) + s := newStorageCleaner(config, component.TelemetrySettings{ + Logger: zaptest.NewLogger(t), + }) require.NotEmpty(t, s.Dependencies()) host := storagetest.NewStorageHost() host.WithExtension(jaegerstorage.ID, &mockStorageExt{ @@ -110,7 +113,9 @@ func TestStorageCleanerExtension(t *testing.T) { func TestGetStorageFactoryError(t *testing.T) { config := &Config{} - s := newStorageCleaner(config, component.TelemetrySettings{}) + s := newStorageCleaner(config, component.TelemetrySettings{ + Logger: zaptest.NewLogger(t), + }) host := storagetest.NewStorageHost() host.WithExtension(jaegerstorage.ID, &mockStorageExt{ name: "storage", @@ -128,6 +133,7 @@ func TestStorageExtensionStartError(t *testing.T) { } var startStatus atomic.Pointer[component.StatusEvent] s := newStorageCleaner(config, component.TelemetrySettings{ + Logger: zaptest.NewLogger(t), ReportStatus: func(status *component.StatusEvent) { startStatus.Store(status) }, From d0e6d353e066b574861c00f34f4358bc41d6181b Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Mon, 15 Jul 2024 22:39:03 -0400 Subject: [PATCH 4/5] fix Signed-off-by: Yuri Shkuro --- .github/workflows/ci-e2e-memory.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-e2e-memory.yaml b/.github/workflows/ci-e2e-memory.yaml index 0726b8cc276..580fb79bc5b 100644 --- a/.github/workflows/ci-e2e-memory.yaml +++ b/.github/workflows/ci-e2e-memory.yaml @@ -1,4 +1,4 @@ -name: CIT Badger +name: CIT Memory on: push: @@ -16,11 +16,8 @@ permissions: # added using https://github.com/step-security/secure-workflows contents: read jobs: - badger: + memory-v2: runs-on: ubuntu-latest - strategy: - matrix: - version: [v1, v2] steps: - name: Harden Runner uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 From dd3a2663fda2d21cf75111e83753861c48587f93 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Mon, 15 Jul 2024 22:47:45 -0400 Subject: [PATCH 5/5] fix Signed-off-by: Yuri Shkuro --- .github/workflows/ci-e2e-memory.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-e2e-memory.yaml b/.github/workflows/ci-e2e-memory.yaml index 580fb79bc5b..3a8dce7d370 100644 --- a/.github/workflows/ci-e2e-memory.yaml +++ b/.github/workflows/ci-e2e-memory.yaml @@ -38,4 +38,4 @@ jobs: uses: ./.github/actions/upload-codecov with: files: cover.out - flags: badger_${{ matrix.version }} + flags: memory_v2