Merge branch 'master' into nikita-tkachenko/gradle-configuration-cache

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 2487cea5160a398549743d2cfd927a863792e3bd
+default_system_tests_commit: &default_system_tests_commit 6998b307ce0e8d206ca6eb99045651dcc8bc853f
 
 parameters:
   nightly:

dd-java-agent/agent-ci-visibility/build.gradle

@@ -73,6 +73,7 @@ excludedClassesCoverage += [
   "datadog.trace.civisibility.source.index.RepoIndexBuilder.RepoIndexingFileVisitor",
   "datadog.trace.civisibility.source.index.RepoIndexFetcher",
   "datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver",
+  "datadog.trace.civisibility.source.Utils",
   "datadog.trace.civisibility.utils.ShellCommandExecutor",
   "datadog.trace.civisibility.utils.ShellCommandExecutor.OutputParser",
   "datadog.trace.civisibility.utils.SpanUtils"

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java

@@ -1,6 +1,7 @@
 package datadog.trace.civisibility;
 
 import datadog.communication.ddagent.SharedCommunicationObjects;
+import datadog.communication.ddagent.TracerVersion;
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.CIVisibility;
 import datadog.trace.api.civisibility.InstrumentationBridge;
@@ -52,6 +53,7 @@
 import datadog.trace.civisibility.source.index.RepoIndexProvider;
 import datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver;
 import datadog.trace.util.Strings;
+import datadog.trace.util.throwable.FatalAgentMisconfigurationError;
 import java.net.InetSocketAddress;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
@@ -74,6 +76,17 @@ public static void start(SharedCommunicationObjects sco) {
       return;
     }
 
+    String injectedTracerVersion = config.getCiVisibilityInjectedTracerVersion();
+    if (injectedTracerVersion != null
+        && !injectedTracerVersion.equals(TracerVersion.TRACER_VERSION)) {
+      throw new FatalAgentMisconfigurationError(
+          "Running JVM with tracer version "
+              + TracerVersion.TRACER_VERSION
+              + " however parent process attempted to inject "
+              + injectedTracerVersion
+              + ". Do not inject the tracer into the forked JVMs manually, or ensure the manually injected version is the same as the one injected automatically");
+    }
+
     sco.createRemaining(config);
 
     GitClient.Factory gitClientFactory = buildGitClientFactory(config);
@@ -134,8 +147,6 @@ private static DDBuildSystemSession.Factory buildSystemSessionFactory(
       CoverageProbeStoreFactory coverageProbeStoreFactory,
       GitClient.Factory gitClientFactory) {
     BackendApiFactory backendApiFactory = new BackendApiFactory(config, sco);
-    BackendApi backendApi = backendApiFactory.createBackendApi();
-
     return (String projectName,
         Path projectRoot,
         String startCommand,
@@ -165,6 +176,7 @@ private static DDBuildSystemSession.Factory buildSystemSessionFactory(
       TestDecorator testDecorator = new TestDecoratorImpl(buildSystemName, ciTags);
       TestModuleRegistry testModuleRegistry = new TestModuleRegistry();
 
+      BackendApi backendApi = backendApiFactory.createBackendApi();
       GitDataUploader gitDataUploader =
           buildGitDataUploader(config, gitInfoProvider, gitClientFactory, backendApi, repoRoot);
       ModuleExecutionSettingsFactory moduleExecutionSettingsFactory =
@@ -228,8 +240,6 @@ private static DDTestFrameworkSession.Factory testFrameworkSessionFactory(
       CoverageProbeStoreFactory coverageProbeStoreFactory,
       GitClient.Factory gitClientFactory) {
     BackendApiFactory backendApiFactory = new BackendApiFactory(config, sco);
-    BackendApi backendApi = backendApiFactory.createBackendApi();
-
     return (String projectName, Path projectRoot, String component, Long startTime) -> {
       CIProviderInfoFactory ciProviderInfoFactory = new CIProviderInfoFactory(config);
       CIProviderInfo ciProviderInfo = ciProviderInfoFactory.createCIProviderInfo(projectRoot);
@@ -275,6 +285,7 @@ private static DDTestFrameworkSession.Factory testFrameworkSessionFactory(
       // either we are in the build system
       // or we are in the tests JVM and the build system is not instrumented
       if (parentProcessSessionId == null || parentProcessModuleId == null) {
+        BackendApi backendApi = backendApiFactory.createBackendApi();
         GitDataUploader gitDataUploader =
             buildGitDataUploader(config, gitInfoProvider, gitClientFactory, backendApi, repoRoot);
         RepoIndexProvider indexProvider = new RepoIndexBuilder(repoRoot, FileSystems.getDefault());

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java

@@ -33,17 +33,19 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
             "Agentless mode is enabled and api key is not set. Please set application key");
       }
       long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis();
-      return new IntakeApi(site, apiKey, timeoutMillis, retryPolicyFactory);
+      String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
+      return new IntakeApi(site, apiKey, traceId, timeoutMillis, retryPolicyFactory);
     }
 
     DDAgentFeaturesDiscovery featuresDiscovery =
         sharedCommunicationObjects.featuresDiscovery(config);
     featuresDiscovery.discoverIfOutdated();
     if (featuresDiscovery.supportsEvpProxy()) {
+      String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
       String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint();
       HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint);
       return new EvpProxyApi(
-          evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient);
+          traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient);
     }
 
     log.warn(

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java

@@ -19,14 +19,21 @@ public class EvpProxyApi implements BackendApi {
 
   private static final String API_VERSION = "v2";
   private static final String X_DATADOG_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain";
+  private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id";
+  private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id";
   private static final String API_SUBDOMAIN = "api";
 
+  private final String traceId;
   private final HttpRetryPolicy.Factory retryPolicyFactory;
   private final HttpUrl evpProxyUrl;
   private final OkHttpClient httpClient;
 
   public EvpProxyApi(
-      HttpUrl evpProxyUrl, HttpRetryPolicy.Factory retryPolicyFactory, OkHttpClient httpClient) {
+      String traceId,
+      HttpUrl evpProxyUrl,
+      HttpRetryPolicy.Factory retryPolicyFactory,
+      OkHttpClient httpClient) {
+    this.traceId = traceId;
     this.evpProxyUrl = evpProxyUrl.resolve(String.format("api/%s/", API_VERSION));
     this.retryPolicyFactory = retryPolicyFactory;
     this.httpClient = httpClient;
@@ -41,6 +48,8 @@ public <T> T post(
         new Request.Builder()
             .url(url)
             .addHeader(X_DATADOG_EVP_SUBDOMAIN_HEADER, API_SUBDOMAIN)
+            .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId)
+            .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId)
             .post(requestBody)
             .build();
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java

@@ -20,15 +20,23 @@ public class IntakeApi implements BackendApi {
 
   private static final String API_VERSION = "v2";
   private static final String DD_API_KEY_HEADER = "dd-api-key";
+  private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id";
+  private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id";
 
   private final String apiKey;
+  private final String traceId;
   private final HttpRetryPolicy.Factory retryPolicyFactory;
   private final HttpUrl hostUrl;
   private final OkHttpClient httpClient;
 
   public IntakeApi(
-      String site, String apiKey, long timeoutMillis, HttpRetryPolicy.Factory retryPolicyFactory) {
+      String site,
+      String apiKey,
+      String traceId,
+      long timeoutMillis,
+      HttpRetryPolicy.Factory retryPolicyFactory) {
     this.apiKey = apiKey;
+    this.traceId = traceId;
     this.retryPolicyFactory = retryPolicyFactory;
 
     final String ciVisibilityAgentlessUrlStr = Config.get().getCiVisibilityAgentlessUrl();
@@ -47,7 +55,12 @@ public <T> T post(
       throws IOException {
     HttpUrl url = hostUrl.resolve(uri);
     Request.Builder requestBuilder =
-        new Request.Builder().url(url).post(requestBody).addHeader(DD_API_KEY_HEADER, apiKey);
+        new Request.Builder()
+            .url(url)
+            .post(requestBody)
+            .addHeader(DD_API_KEY_HEADER, apiKey)
+            .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId)
+            .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId);
 
     Request request = requestBuilder.build();
     HttpRetryPolicy retryPolicy = retryPolicyFactory.create();

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java

@@ -1,5 +1,6 @@
 package datadog.trace.civisibility.config;
 
+import datadog.communication.ddagent.TracerVersion;
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.config.Configurations;
 import datadog.trace.api.civisibility.config.ModuleExecutionSettings;
@@ -156,6 +157,11 @@ private Map<String, String> getPropertiesPropagatedToChildProcess(
             CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED),
         Boolean.toString(false));
 
+    propagatedSystemProperties.put(
+        Strings.propertyNameToSystemPropertyName(
+            CiVisibilityConfig.CIVISIBILITY_INJECTED_TRACER_VERSION),
+        TracerVersion.TRACER_VERSION);
+
     return propagatedSystemProperties;
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializer.java

@@ -26,10 +26,10 @@ public static ByteBuffer serialize(Collection<SkippableTest> skippableTests) {
       String name = test.getName();
       String parameters = test.getParameters();
 
-      length += suite.length();
-      length += name.length();
+      length += suite.getBytes(CHARSET).length;
+      length += name.getBytes(CHARSET).length;
       if (parameters != null) {
-        length += parameters.length();
+        length += parameters.getBytes(CHARSET).length;
       }
     }
 
@@ -41,14 +41,18 @@ public static ByteBuffer serialize(Collection<SkippableTest> skippableTests) {
       String name = test.getName();
       String parameters = test.getParameters();
 
-      buffer.putInt(suite.length());
-      buffer.put(suite.getBytes(CHARSET));
-      buffer.putInt(name.length());
-      buffer.put(name.getBytes(CHARSET));
+      byte[] suiteBytes = suite.getBytes(CHARSET);
+      buffer.putInt(suiteBytes.length);
+      buffer.put(suiteBytes);
+
+      byte[] nameBytes = name.getBytes(CHARSET);
+      buffer.putInt(nameBytes.length);
+      buffer.put(nameBytes);
 
       if (parameters != null) {
-        buffer.putInt(parameters.length());
-        buffer.put(parameters.getBytes(CHARSET));
+        byte[] parametersBytes = parameters.getBytes(CHARSET);
+        buffer.putInt(parametersBytes.length);
+        buffer.put(parametersBytes);
       } else {
         buffer.putInt(-1);
       }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/Utils.java

@@ -7,7 +7,26 @@ public abstract class Utils {
 
   public static InputStream getClassStream(Class<?> clazz) throws IOException {
     String className = clazz.getName();
-    String classPath = "/" + className.replace('.', '/') + ".class";
-    return clazz.getResourceAsStream(classPath);
+    InputStream classStream = clazz.getResourceAsStream(toResourceName(className));
+    if (classStream != null) {
+      return classStream;
+    } else {
+      // might be auto-generated inner class (e.g. Mockito mock)
+      String topLevelClassName = stripNestedClassNames(clazz.getName());
+      return clazz.getResourceAsStream(toResourceName(topLevelClassName));
+    }
+  }
+
+  private static String toResourceName(String className) {
+    return "/" + className.replace('.', '/') + ".class";
+  }
+
+  public static String stripNestedClassNames(String className) {
+    int innerClassNameIdx = className.indexOf('$');
+    if (innerClassNameIdx >= 0) {
+      return className.substring(0, innerClassNameIdx);
+    } else {
+      return className;
+    }
   }
 }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndex.java

@@ -50,7 +50,7 @@ public List<String> getRootPackages() {
 
   @Nullable
   public String getSourcePath(@Nonnull Class<?> c) {
-    String topLevelClassName = stripNestedClassNames(c.getName());
+    String topLevelClassName = Utils.stripNestedClassNames(c.getName());
     SourceType sourceType = detectSourceType(c);
     String extension = sourceType.getExtension();
     String classNameWithExtension = topLevelClassName + extension;
@@ -107,15 +107,6 @@ private SourceType detectSourceType(Class<?> c) {
     return SourceType.JAVA;
   }
 
-  private String stripNestedClassNames(String className) {
-    int innerClassNameIdx = className.indexOf('$');
-    if (innerClassNameIdx >= 0) {
-      return className.substring(0, innerClassNameIdx);
-    } else {
-      return className;
-    }
-  }
-
   /**
    * Names of package-private classes do not have to correspond to the names of their source code
    * files. For such classes filename is extracted from SourceFile attribute that is available in
@@ -126,6 +117,10 @@ private String getSourcePathForPackagePrivateOrNonJavaClass(Class<?> c) {
       SourceFileAttributeVisitor sourceFileAttributeVisitor = new SourceFileAttributeVisitor();
 
       try (InputStream classStream = Utils.getClassStream(c)) {
+        if (classStream == null) {
+          log.debug("Could not get input stream for class {}", c.getName());
+          return null;
+        }
         ClassReader classReader = new ClassReader(classStream);
         classReader.accept(
             sourceFileAttributeVisitor, ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES);

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy

@@ -135,10 +135,11 @@ class ConfigurationApiImplTest extends Specification {
   }
 
   private BackendApi givenEvpProxy() {
+    String traceId = "a-trace-id"
     HttpUrl proxyUrl = HttpUrl.get(intakeServer.address)
     HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0)
     OkHttpClient client = OkHttpUtils.buildHttpClient(proxyUrl, REQUEST_TIMEOUT_MILLIS)
-    return new EvpProxyApi(proxyUrl, retryPolicyFactory, client)
+    return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client)
   }
 
   private static TracerEnvironment givenTracerEnvironment() {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy → dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy

@@ -22,6 +22,8 @@ class SkippableTestsSerializerTest extends Specification {
     tests << [
       // single test
       [["suite", "name", null]],
+      [["suite", "𝕄 add user properties 𝕎 addUserProperties()", null]],
+      // non-ASCII characters
       [["suite", "name", "parameters"]],
       [["suite", "name", "{\"metadata\":{\"test_name\":\"test display name with #a #b #c\"}}"]],
       // multiple tests

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy

@@ -114,10 +114,11 @@ class GitDataApiTest extends Specification {
   }
 
   private BackendApi givenEvpProxy() {
+    String traceId = "a-trace-id"
     HttpUrl proxyUrl = HttpUrl.get(intakeServer.address)
     HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0)
     OkHttpClient client = OkHttpUtils.buildHttpClient(proxyUrl, REQUEST_TIMEOUT_MILLIS)
-    return new EvpProxyApi(proxyUrl, retryPolicyFactory, client)
+    return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client)
   }
 
   private Path givenPackFile() {

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/RepoIndexSourcePathResolverTest.groovy

@@ -72,6 +72,17 @@ class RepoIndexSourcePathResolverTest extends Specification {
     sourcePathResolver.getSourcePath(PackagePrivateClass) == expectedSourcePath
   }
 
+  def "test source path resolution for class nested into package-private class"() {
+    setup:
+    def expectedSourcePath = givenSourceFile(RepoIndexSourcePathResolverTest, repoRoot + "/src")
+
+    when:
+    def sourcePathResolver = new RepoIndexSourcePathResolver(repoRoot, packageResolver, fileSystem)
+
+    then:
+    sourcePathResolver.getSourcePath(PackagePrivateClass.NestedIntoPackagePrivateClass) == expectedSourcePath
+  }
+
   def "test source path resolution for non-java class whose file name is different from class name"() {
     setup:
     def expectedSourcePath = givenSourceFile(RepoIndexSourcePathResolverTest, repoRoot + "/src")
@@ -173,6 +184,7 @@ class RepoIndexSourcePathResolverTest extends Specification {
 
 @PackageScope
 class PackagePrivateClass {
+  class NestedIntoPackagePrivateClass {}
 }
 
 class PublicClassWhoseNameDoesNotCorrespondToFileName {

dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanProbesIntegrationTest.java

@@ -58,7 +58,12 @@ void testLineRangeSpan() throws Exception {
   void testSingleLineSpan() throws Exception {
     final String METHOD_NAME = "fullMethod";
     final String EXPECTED_UPLOADS = "2"; // 2 probe statuses: RECEIVED + ERROR
-    SpanProbe spanProbe = SpanProbe.builder().probeId(PROBE_ID).where(MAIN_CLASS_NAME, 80).build();
+    SpanProbe spanProbe =
+        SpanProbe.builder()
+            .probeId(PROBE_ID)
+            // on line: System.out.println("fullMethod");
+            .where(MAIN_CLASS_NAME, 88)
+            .build();
     setCurrentConfiguration(createSpanConfig(spanProbe));
     targetProcess = createProcessBuilder(logFilePath, METHOD_NAME, EXPECTED_UPLOADS).start();
     AtomicBoolean received = new AtomicBoolean(false);

dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java

@@ -60,6 +60,8 @@ public final class CiVisibilityConfig {
       "civisibility.module.execution.settings.cache.size";
   public static final String CIVISIBILITY_COVERAGE_SEGMENTS_ENABLED =
       "civisibility.coverage.segments.enabled";
+  public static final String CIVISIBILITY_INJECTED_TRACER_VERSION =
+      "civisibility.injected.tracer.version";
 
   private CiVisibilityConfig() {}
 }