Merge branch 'master' into mhlidd/client_error_statuses

DataDog · Oct 9, 2024 · a3e22e1 · a3e22e1
2 parents bd83253 + 14a45ea
commit a3e22e1
Show file tree

Hide file tree

Showing 81 changed files with 6,777 additions and 3,088 deletions.
diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2
@@ -782,6 +782,7 @@ jobs:
           name: Gather muzzle tasks
           command: >-
             SKIP_BUILDSCAN="true"
+            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
             ./gradlew writeMuzzleTasksToFile
             << pipeline.parameters.gradle_flags >>
             --max-workers=3
@@ -1217,6 +1218,20 @@ build_test_jobs: &build_test_jobs
       parallelism: 4
       maxWorkers: 4
       testJvm: "8"
+
+  - tests:
+      requires:
+        - ok_to_test
+      name: z_test_8_flaky_debugger
+      gradleTarget: ":debuggerTest"
+      gradleParameters: "-PrunFlakyTests"
+      continueOnFailure: true
+      triggeredBy: *debugger_modules
+      stage: debugger
+      cacheType: base
+      parallelism: 4
+      maxWorkers: 4
+      testJvm: "8"
 {% endif %}
 
   - tests:
@@ -1420,6 +1435,7 @@ build_test_jobs: &build_test_jobs
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}"
 {% endfor %}
+        - muzzle
         - profiling
         - debugger
         - system-tests

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -216,7 +216,7 @@ package-oci:
 onboarding_tests_installer:
   parallel:
     matrix:
-      - ONBOARDING_FILTER_WEBLOG: [test-app-java, test-app-java-container, test-app-java-container-jdk15, test-app-java-alpine-libgcc]
+      - ONBOARDING_FILTER_WEBLOG: [test-app-java, test-app-java-container, test-app-java-container-jdk15, test-app-java-alpine]
         SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION, SIMPLE_AUTO_INJECTION_PROFILING ]
       - ONBOARDING_FILTER_WEBLOG: [test-app-java-buildpack]
         SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION ]

diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java
@@ -3,11 +3,13 @@
 import datadog.communication.ddagent.DDAgentFeaturesDiscovery;
 import datadog.communication.ddagent.SharedCommunicationObjects;
 import datadog.communication.http.HttpRetryPolicy;
+import datadog.communication.http.OkHttpUtils;
 import datadog.trace.api.Config;
 import datadog.trace.util.throwable.FatalAgentMisconfigurationError;
 import java.util.function.Function;
 import javax.annotation.Nullable;
 import okhttp3.HttpUrl;
+import okhttp3.OkHttpClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,13 +36,10 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
             "Agentless mode is enabled and api key is not set. Please set application key");
       }
       String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
-      return new IntakeApi(
-          agentlessUrl,
-          apiKey,
-          traceId,
-          retryPolicyFactory,
-          sharedCommunicationObjects.okHttpClient,
-          true);
+      OkHttpClient httpClient =
+          OkHttpUtils.buildHttpClient(
+              agentlessUrl, config.getCiVisibilityBackendApiTimeoutMillis());
+      return new IntakeApi(agentlessUrl, apiKey, traceId, retryPolicyFactory, httpClient, true);
     }
 
     DDAgentFeaturesDiscovery featuresDiscovery =

diff --git a/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java b/communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java
@@ -36,21 +36,11 @@ public void createRemaining(Config config) {
       }
     }
     if (okHttpClient == null) {
-      String unixDomainSocket;
-      String namedPipe;
-      HttpUrl url;
-      if (!config.isCiVisibilityAgentlessEnabled()) {
-        unixDomainSocket = SocketUtils.discoverApmSocket(config);
-        namedPipe = config.getAgentNamedPipe();
-        url = agentUrl;
-      } else {
-        unixDomainSocket = null;
-        namedPipe = null;
-        url = null;
-      }
+      String unixDomainSocket = SocketUtils.discoverApmSocket(config);
+      String namedPipe = config.getAgentNamedPipe();
       okHttpClient =
           OkHttpUtils.buildHttpClient(
-              url, unixDomainSocket, namedPipe, getHttpClientTimeout(config));
+              agentUrl, unixDomainSocket, namedPipe, getHttpClientTimeout(config));
     }
   }
 

diff --git a/dd-java-agent/agent-ci-visibility/build.gradle b/dd-java-agent/agent-ci-visibility/build.gradle
@@ -25,8 +25,8 @@ dependencies {
   api libs.slf4j
 
   implementation libs.bundles.asm
-  implementation group: 'org.jacoco', name: 'org.jacoco.core', version: '0.8.9'
-  implementation group: 'org.jacoco', name: 'org.jacoco.report', version: '0.8.9'
+  implementation group: 'org.jacoco', name: 'org.jacoco.core', version: '0.8.12'
+  implementation group: 'org.jacoco', name: 'org.jacoco.report', version: '0.8.12'
 
   implementation project(':communication')
   implementation project(':internal-api')

diff --git a/...ent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/BuildSystemSession.java b/...ent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/BuildSystemSession.java
@@ -3,6 +3,7 @@
 import datadog.trace.api.civisibility.domain.BuildModuleLayout;
 import datadog.trace.api.civisibility.domain.BuildSessionSettings;
 import datadog.trace.api.civisibility.domain.JavaAgent;
+import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.civisibility.config.JvmInfo;
 import java.nio.file.Path;
 import java.util.Collection;
@@ -25,6 +26,8 @@ BuildSystemModule testModuleStart(
       @Nullable Collection<Path> classpath,
       @Nullable JavaAgent jacocoAgent);
 
+  AgentSpan testTaskStart(String taskName);
+
   BuildSessionSettings getSettings();
 
   interface Factory {

diff --git a/...y/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java b/...y/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java
@@ -1,5 +1,7 @@
 package datadog.trace.civisibility.domain.buildsystem;
 
+import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan;
+
 import datadog.trace.api.Config;
 import datadog.trace.api.DDTags;
 import datadog.trace.api.civisibility.CIConstants;
@@ -202,6 +204,11 @@ public BuildSystemModuleImpl testModuleStart(
         this::onModuleFinish);
   }
 
+  @Override
+  public AgentSpan testTaskStart(String taskName) {
+    return startSpan("ci_visibility", taskName, span.context());
+  }
+
   private void onModuleFinish(AgentSpan moduleSpan) {
     // multiple modules can finish in parallel
     synchronized (tagPropagationLock) {

diff --git a/...ci-visibility/src/main/java/datadog/trace/civisibility/events/BuildEventsHandlerImpl.java b/...ci-visibility/src/main/java/datadog/trace/civisibility/events/BuildEventsHandlerImpl.java
@@ -5,6 +5,7 @@
 import datadog.trace.api.civisibility.domain.BuildSessionSettings;
 import datadog.trace.api.civisibility.domain.JavaAgent;
 import datadog.trace.api.civisibility.events.BuildEventsHandler;
+import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.Tags;
 import datadog.trace.civisibility.config.JvmInfo;
 import datadog.trace.civisibility.config.JvmInfoFactory;
@@ -23,7 +24,10 @@ public class BuildEventsHandlerImpl<SessionKey> implements BuildEventsHandler<Se
   private final ConcurrentMap<SessionKey, BuildSystemSession> inProgressTestSessions =
       new ConcurrentHashMap<>();
 
-  private final ConcurrentMap<TestModuleDescriptor<SessionKey>, BuildSystemModule>
+  private final ConcurrentMap<BuildTaskDescriptor<SessionKey>, AgentSpan> inProgressBuildTasks =
+      new ConcurrentHashMap<>();
+
+  private final ConcurrentMap<BuildTaskDescriptor<SessionKey>, BuildSystemModule>
       inProgressTestModules = new ConcurrentHashMap<>();
 
   private final BuildSystemSession.Factory sessionFactory;
@@ -79,6 +83,47 @@ public void onTestSessionFinish(final SessionKey sessionKey) {
     testSession.end(null);
   }
 
+  @Override
+  public void onBuildTaskStart(
+      SessionKey sessionKey, String taskName, Map<String, Object> additionalTags) {
+    BuildSystemSession testSession = inProgressTestSessions.get(sessionKey);
+    if (testSession == null) {
+      throw new IllegalStateException("Could not find session span for key: " + sessionKey);
+    }
+    AgentSpan buildTask = testSession.testTaskStart(taskName);
+    for (Map.Entry<String, Object> e : additionalTags.entrySet()) {
+      buildTask.setTag(e.getKey(), e.getValue());
+    }
+    inProgressBuildTasks.put(new BuildTaskDescriptor<>(sessionKey, taskName), buildTask);
+  }
+
+  @Override
+  public void onBuildTaskFail(SessionKey sessionKey, String taskName, Throwable throwable) {
+    AgentSpan buildTask = inProgressBuildTasks.get(new BuildTaskDescriptor<>(sessionKey, taskName));
+    if (buildTask == null) {
+      throw new IllegalStateException(
+          "Could not find build task span for session key "
+              + sessionKey
+              + " and task name "
+              + taskName);
+    }
+    buildTask.setError(true);
+    buildTask.addThrowable(throwable);
+  }
+
+  @Override
+  public void onBuildTaskFinish(SessionKey sessionKey, String taskName) {
+    AgentSpan buildTask = inProgressBuildTasks.get(new BuildTaskDescriptor<>(sessionKey, taskName));
+    if (buildTask == null) {
+      throw new IllegalStateException(
+          "Could not find build task span for session key "
+              + sessionKey
+              + " and task name "
+              + taskName);
+    }
+    buildTask.finish();
+  }
+
   @Override
   public BuildModuleSettings onTestModuleStart(
       final SessionKey sessionKey,
@@ -104,8 +149,8 @@ public BuildModuleSettings onTestModuleStart(
       }
     }
 
-    TestModuleDescriptor<SessionKey> testModuleDescriptor =
-        new TestModuleDescriptor<>(sessionKey, moduleName);
+    BuildTaskDescriptor<SessionKey> testModuleDescriptor =
+        new BuildTaskDescriptor<>(sessionKey, moduleName);
     inProgressTestModules.put(testModuleDescriptor, testModule);
 
     return testModule.getSettings();
@@ -126,8 +171,8 @@ public void onTestModuleFail(
   }
 
   private BuildSystemModule getTestModule(final SessionKey sessionKey, final String moduleName) {
-    TestModuleDescriptor<SessionKey> testModuleDescriptor =
-        new TestModuleDescriptor<>(sessionKey, moduleName);
+    BuildTaskDescriptor<SessionKey> testModuleDescriptor =
+        new BuildTaskDescriptor<>(sessionKey, moduleName);
     BuildSystemModule testModule = inProgressTestModules.get(testModuleDescriptor);
     if (testModule == null) {
       throw new IllegalStateException(
@@ -138,8 +183,8 @@ private BuildSystemModule getTestModule(final SessionKey sessionKey, final Strin
 
   @Override
   public void onTestModuleFinish(SessionKey sessionKey, String moduleName) {
-    TestModuleDescriptor<SessionKey> testModuleDescriptor =
-        new TestModuleDescriptor<>(sessionKey, moduleName);
+    BuildTaskDescriptor<SessionKey> testModuleDescriptor =
+        new BuildTaskDescriptor<>(sessionKey, moduleName);
     BuildSystemModule testModule = inProgressTestModules.remove(testModuleDescriptor);
     if (testModule == null) {
       throw new IllegalStateException(

diff --git a/...nt-ci-visibility/src/main/java/datadog/trace/civisibility/events/BuildTaskDescriptor.java b/...nt-ci-visibility/src/main/java/datadog/trace/civisibility/events/BuildTaskDescriptor.java
@@ -0,0 +1,31 @@
+package datadog.trace.civisibility.events;
+
+import java.util.Objects;
+
+public class BuildTaskDescriptor<T> {
+
+  private final T sessionKey;
+  private final String taskName;
+
+  public BuildTaskDescriptor(T sessionKey, String taskName) {
+    this.sessionKey = sessionKey;
+    this.taskName = taskName;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    BuildTaskDescriptor<?> that = (BuildTaskDescriptor<?>) o;
+    return sessionKey.equals(that.sessionKey) && taskName.equals(that.taskName);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(sessionKey, taskName);
+  }
+}
diff --git a/...t-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestModuleDescriptor.java b/...t-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestModuleDescriptor.java
diff --git a/...nstrumentation/avro/src/main/java/datadog/trace/instrumentation/avro/SchemaExtractor.java b/...nstrumentation/avro/src/main/java/datadog/trace/instrumentation/avro/SchemaExtractor.java
@@ -32,7 +32,10 @@ public static boolean extractProperty(
 
     switch (field.schema().getType()) {
       case RECORD:
-        type = "object";
+        type = "#/components/schemas/" + field.schema().getFullName();
+        if (!extractSchema(field.schema(), builder, depth)) {
+          return false;
+        }
         break;
       case ENUM:
         type = "string";
@@ -41,10 +44,26 @@ public static boolean extractProperty(
       case ARRAY:
         array = true;
         type = getType(field.schema().getElementType().getType().getName());
+        if (type == "record") {
+          type = "#/components/schemas/" + field.schema().getElementType().getFullName();
+          if (!extractSchema(field.schema().getElementType(), builder, depth)) {
+            return false;
+          }
+          ;
+        }
         break;
       case MAP:
         type = "object";
-        description = "Map type";
+        String keys = "string";
+        String values = getType(field.schema().getValueType().getType().getName());
+        if (values == "record") {
+          values = "#/components/schemas/" + field.schema().getValueType().getFullName();
+          if (!extractSchema(field.schema().getValueType(), builder, depth)) {
+            return false;
+          }
+          ;
+        }
+        description = "Map type with " + keys + " keys and " + values + " values";
         break;
       case STRING:
         type = "string";
@@ -164,6 +183,8 @@ private static String getType(String type) {
         return "boolean";
       case "null":
         return "null";
+      case "record":
+        return "record";
       default:
         return "string";
     }