[CI][WIP] Scrape metrics at the end of e2e test

.github/workflows/ci-e2e-all.yml

@@ -36,7 +36,3 @@ jobs:
   opensearch:
     uses: ./.github/workflows/ci-e2e-opensearch.yml
 
-
-
-
-

.github/workflows/ci-e2e-badger.yaml

@@ -40,6 +40,14 @@ jobs:
             STORAGE=badger make jaeger-v2-storage-integration-test
             ;;
         esac
+
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ matrix.version }} == "v2"
+      with:
+            name: badger-metrics-${{ matrix.version }}
+            path: ./metrics/badger_metrics.txt
+            retention-days: 7
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-cassandra.yml

@@ -48,6 +48,14 @@ jobs:
       run: bash scripts/cassandra-integration-test.sh ${{ matrix.version.major }} ${{ matrix.version.schema }} ${{ matrix.jaeger-version }}
       env:
         SKIP_APPLY_SCHEMA: ${{ matrix.create-schema == 'auto' && true || false }}
+
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ matrix.jaeger-version }} == "v2"
+      with:
+          name: cassandra_metrics_${{ matrix.version.major }}_${{ matrix.version.schema }}_${{ matrix.jaeger-version }}
+          path: ./metrics/cassandra_metrics.txt
+          retention-days: 7
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-elasticsearch.yml

@@ -58,7 +58,15 @@ jobs:
     - name: Run ${{ matrix.version.distribution }} integration tests
       id: test-execution
       run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
-
+
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ matrix.version.jaeger }} == "v2"
+      with:
+            name: elasticsearch-metrics-${{ matrix.version.major }}_${{ matrix.version.jaeger}}
+            path: ./metrics/elasticsearch_metrics.txt
+            retention-days: 7
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-grpc.yml

@@ -40,6 +40,14 @@ jobs:
             STORAGE=grpc SPAN_STORAGE_TYPE=memory make jaeger-v2-storage-integration-test
             ;;
         esac
+
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ matrix.version }} == "v2"
+      with:
+              name: grpc-metrics-${{ matrix.version }}
+              path: ./metrics/grpc_metrics.txt
+              retention-days: 7
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-kafka.yml

@@ -34,6 +34,14 @@ jobs:
     - name: Run Kafka integration tests
       id: test-execution
       run: bash scripts/kafka-integration-test.sh -j ${{ matrix.jaeger-version }}
+
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ matrix.jaeger-version }} == "v2"
+      with:
+              name: kafka-metrics-${{ matrix.jaeger-version }}
+              path: ./metrics/kafka_metrics.txt
+              retention-days: 7
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-memory.yaml

@@ -29,7 +29,14 @@ jobs:
     - name: Run Memory storage integration tests
       run: |
         STORAGE=memory_v2 make jaeger-v2-storage-integration-test
-
+
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@v4
+      with:
+          name: memory-metrics
+          path: ./metrics/memory_metrics.txt
+          retention-days: 7
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-opensearch.yml

@@ -52,6 +52,14 @@ jobs:
       id: test-execution
       run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
 
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ matrix.version.major }} == "v2"
+      with:
+          name: opensearch-metrics-${{ matrix.version.jaeger }}
+          path: ./metrics/opensearch_metrics.txt
+          retention-days: 7
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.gitignore

@@ -50,3 +50,4 @@ sha256sum.combined.txt
 resource.syso
 .gocache
 test-results.json
+metrics/

cmd/jaeger/internal/integration/e2e_integration.go

@@ -153,6 +153,7 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) {
 	require.NoError(t, err)
 
 	t.Cleanup(func() {
+		scrapeMetrics(t, storage)
 		// Call e2eCleanUp to close the SpanReader and SpanWriter gRPC connection.
 		s.e2eCleanUp(t)
 	})
@@ -211,6 +212,30 @@ func (s *E2EStorageIntegration) e2eCleanUp(t *testing.T) {
 	require.NoError(t, s.SpanWriter.(io.Closer).Close())
 }
 
+func scrapeMetrics(t *testing.T, storage string) {
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost:8888/metrics", nil)
+	require.NoError(t, err)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	err = os.MkdirAll("../../../../metrics", os.ModePerm)
+	if err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+
+	metricsFile, err := os.Create(fmt.Sprintf("../../../../metrics/%v_metrics.txt", storage))
+	if err != nil {
+		t.Fatalf("Failed to create metrics file: %v", err)
+	}
+	defer metricsFile.Close()
+
+	_, err = io.Copy(metricsFile, resp.Body)
+	require.NoError(t, err)
+}
+
 func createStorageCleanerConfig(t *testing.T, configFile string, storage string) string {
 	data, err := os.ReadFile(configFile)
 	require.NoError(t, err)