Merge remote-tracking branch 'origin/master' into vandonr/payload

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 74de81e97fd6786854c2fe254764cfb75610c676
+default_system_tests_commit: &default_system_tests_commit edfea31b7a9ceaed03b705de34a4e525853444c0
 
 parameters:
   nightly:
@@ -635,7 +635,7 @@ jobs:
     <<: *tests
 
     docker:
-      - image: << pipeline.parameters.docker_image >>:<< parameters.testJvm >>
+      - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}<< parameters.testJvm >>
         environment:
           - CI_USE_TEST_AGENT=true
       - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
@@ -676,7 +676,7 @@ jobs:
     resource_class: medium
 
     docker:
-      - image: << pipeline.parameters.docker_image >>:7
+      - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}8
       - image: datadog/agent:7.34.0
         environment:
           - DD_APM_ENABLED=true
@@ -687,7 +687,7 @@ jobs:
     <<: *defaults
     resource_class: medium
     docker:
-      - image: << pipeline.parameters.docker_image >>:7
+      - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}7
 
     steps:
       - setup_code

.circleci/start_docker_autoforward.sh

@@ -1,9 +1,6 @@
 #!/usr/bin/env bash
 set -eux
 
-# workaround for https://github.com/docker/docker-py/issues/3113
-pip3 install --force-reinstall "urllib3<2"
-
 if [[ -n "${DOCKER_CERT_PATH:-}" ]]; then
     TLS_ARGS="
         --secure

.github/CODEOWNERS

@@ -33,11 +33,10 @@ dd-java-agent/agent-debugger/ @DataDog/debugger-java
 dd-smoke-tests/debugger-integration-tests/ @DataDog/debugger-java
 
 # @DataDog/asm-java (AppSec/IAST)
-dd-java-agent/appsec/                  @DataDog/asm-java
 dd-java-agent/agent-iast/              @DataDog/asm-java
 dd-java-agent/instrumentation/*iast*   @DataDog/asm-java
 dd-java-agent/instrumentation/*appsec* @DataDog/asm-java
-dd-smoke-tests/appsec/                 @DataDog/asm-java
+**/appsec/                             @DataDog/asm-java
 **/iast/                               @DataDog/asm-java
 **/Iast*.java                          @DataDog/asm-java
 **/Iast*.groovy                        @DataDog/asm-java

CONTRIBUTING.md

@@ -200,8 +200,6 @@ Suggested plugins and settings:
   * With java use the following import layout (groovy should still use the default) to ensure consistency with google-java-format:
     ![import layout](https://user-images.githubusercontent.com/734411/43430811-28442636-94ae-11e8-86f1-f270ddcba023.png)
 * [Google Java Format](https://plugins.jetbrains.com/plugin/8527-google-java-format)
-* [Save Actions](https://plugins.jetbrains.com/plugin/7642-save-actions)
-  ![Recommended Settings](https://user-images.githubusercontent.com/35850765/124003079-838f4280-d9a4-11eb-9250-5c517631e362.png)
 
 ## Troubleshooting
 

communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java

@@ -101,9 +101,7 @@ private void doConnect() {
         if (log.isDebugEnabled()) {
           log.debug("Creating StatsD client - {}", statsDAddress());
         }
-        // when using UDS, set "entity-id" to "none" to avoid having the DogStatsD
-        // server add origin tags (see https://github.com/DataDog/jmxfetch/pull/264)
-        String entityID = port == 0 ? "none" : null;
+
         NonBlockingStatsDClientBuilder clientBuilder =
             new NonBlockingStatsDClientBuilder()
                 .threadFactory(STATSD_CLIENT_THREAD_FACTORY)
@@ -112,12 +110,53 @@ private void doConnect() {
                 .hostname(host)
                 .port(port)
                 .namedPipe(namedPipe)
-                .errorHandler(this)
-                .entityID(entityID);
+                .errorHandler(this);
+
+        // when using UDS, set "entity-id" to "none" to avoid having the DogStatsD
+        // server add origin tags (see https://github.com/DataDog/jmxfetch/pull/264)
+        if (this.port == 0) {
+          clientBuilder.constantTags("dd.internal.card:none");
+          clientBuilder.entityID("none");
+        } else {
+          clientBuilder.entityID(null);
+        }
+
+        Integer queueSize = Config.get().getStatsDClientQueueSize();
+        if (queueSize != null) {
+          clientBuilder.queueSize(queueSize);
+        }
+
         // when using UDS set the datagram size to 8k (2k on Mac due to lower OS default)
+        // but also make sure packet size isn't larger than the configured socket buffer
         if (this.port == 0) {
-          clientBuilder.maxPacketSizeBytes(Platform.isMac() ? 2048 : 8192);
+          Integer timeout = Config.get().getStatsDClientSocketTimeout();
+          if (timeout != null) {
+            clientBuilder.timeout(timeout);
+          }
+          Integer bufferSize = Config.get().getStatsDClientSocketBuffer();
+          if (bufferSize != null) {
+            clientBuilder.socketBufferSize(bufferSize);
+          }
+          int packetSize = Platform.isMac() ? 2048 : 8192;
+          if (bufferSize != null && bufferSize < packetSize) {
+            packetSize = bufferSize;
+          }
+          clientBuilder.maxPacketSizeBytes(packetSize);
+        }
+
+        if (log.isDebugEnabled()) {
+          if (this.port == 0) {
+            log.debug(
+                "Configured StatsD client - queueSize={}, maxPacketSize={}, socketBuffer={}, socketTimeout={}",
+                clientBuilder.queueSize,
+                clientBuilder.maxPacketSizeBytes,
+                clientBuilder.socketBufferSize,
+                clientBuilder.timeout);
+          } else {
+            log.debug("Configured StatsD client - queueSize={}", clientBuilder.queueSize);
+          }
         }
+
         try {
           statsd = clientBuilder.build();
           if (log.isDebugEnabled()) {

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecorator.java

@@ -72,6 +72,10 @@ public AgentSpan onConnection(final AgentSpan span, final CONNECTION connection)
       CharSequence hostName = dbHostname(connection);
       if (hostName != null) {
         span.setTag(Tags.PEER_HOSTNAME, hostName);
+
+        if (Config.get().isDbClientSplitByHost()) {
+          span.setServiceName(hostName.toString());
+        }
       }
     }
     return span;

dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecoratorTest.groovy

@@ -4,6 +4,7 @@ import datadog.trace.api.DDTags
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan
 import datadog.trace.bootstrap.instrumentation.api.Tags
 
+import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_HOST
 import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE
 import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE_TYPE_SUFFIX
 
@@ -35,8 +36,9 @@ class DatabaseClientDecoratorTest extends ClientDecoratorTest {
 
   def "test onConnection"() {
     setup:
-    injectSysConfig(DB_CLIENT_HOST_SPLIT_BY_INSTANCE, "$renameService")
-    injectSysConfig(DB_CLIENT_HOST_SPLIT_BY_INSTANCE_TYPE_SUFFIX, "$typeSuffix")
+    injectSysConfig(DB_CLIENT_HOST_SPLIT_BY_INSTANCE, "$renameByInstance")
+    injectSysConfig(DB_CLIENT_HOST_SPLIT_BY_INSTANCE_TYPE_SUFFIX, "$instanceTypeSuffix")
+    injectSysConfig(DB_CLIENT_HOST_SPLIT_BY_HOST, "$renameByHost")
     def decorator = newDecorator()
 
     when:
@@ -49,24 +51,34 @@ class DatabaseClientDecoratorTest extends ClientDecoratorTest {
       if (session.hostname != null) {
         1 * span.setTag(Tags.PEER_HOSTNAME, session.hostname)
       }
-      if (typeSuffix && renameService && session.instance) {
+      if (instanceTypeSuffix && renameByInstance && session.instance) {
         1 * span.setServiceName(session.instance + "-" + decorator.dbType())
-      } else if (renameService && session.instance) {
+      } else if (renameByInstance && session.instance) {
         1 * span.setServiceName(session.instance)
+      } else if (renameByHost) {
+        1 * span.setServiceName(session.hostname)
       }
     }
     0 * _
 
     where:
-    renameService | typeSuffix | session
-    false         | false      | null
-    true          | false      | [user: "test-user", hostname: "test-hostname"]
-    false         | false      | [instance: "test-instance", hostname: "test-hostname"]
-    true          | false      | [user: "test-user", instance: "test-instance"]
-    false         | true       | null
-    true          | true       | [user: "test-user", hostname: "test-hostname"]
-    false         | true       | [instance: "test-instance", hostname: "test-hostname"]
-    true          | true       | [user: "test-user", instance: "test-instance"]
+    renameByInstance | instanceTypeSuffix | renameByHost  | session
+    false            | false              | false         | null
+    true             | false              | false         | [user: "test-user", hostname: "test-hostname"]
+    false            | false              | false         | [instance: "test-instance", hostname: "test-hostname"]
+    true             | false              | false         | [user: "test-user", instance: "test-instance"]
+    false            | true               | false         | null
+    true             | true               | false         | [user: "test-user", hostname: "test-hostname"]
+    false            | true               | false         | [instance: "test-instance", hostname: "test-hostname"]
+    true             | true               | false         | [user: "test-user", instance: "test-instance"]
+    false            | false              | true          | null
+    true             | false              | true          | [user: "test-user", hostname: "test-hostname"]
+    false            | false              | true          | [instance: "test-instance", hostname: "test-hostname"]
+    true             | false              | true          | [user: "test-user", instance: "test-instance"]
+    false            | true               | true          | null
+    true             | true               | true          | [user: "test-user", hostname: "test-hostname"]
+    false            | true               | true          | [instance: "test-instance", hostname: "test-hostname"]
+    true             | true               | true          | [user: "test-user", instance: "test-instance"]
   }
 
   def "test onStatement"() {

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/AwsCodePipelineInfo.java

@@ -0,0 +1,29 @@
+package datadog.trace.civisibility.ci;
+
+import datadog.trace.api.git.GitInfo;
+
+class AwsCodePipelineInfo implements CIProviderInfo {
+
+  public static final String AWS_CODEPIPELINE = "CODEBUILD_INITIATOR";
+  public static final String AWS_CODEPIPELINE_PROVIDER_NAME = "awscodepipeline";
+  public static final String AWS_CODEPIPELINE_EXECUTION_ID = "DD_PIPELINE_EXECUTION_ID";
+  public static final String AWS_CODEPIPELINE_ACTION_EXECUTION_ID = "DD_ACTION_EXECUTION_ID";
+  public static final String AWS_CODEPIPELINE_ARN = "CODEBUILD_BUILD_ARN";
+
+  @Override
+  public GitInfo buildCIGitInfo() {
+    return GitInfo.NOOP;
+  }
+
+  @Override
+  public CIInfo buildCIInfo() {
+    return CIInfo.builder()
+        .ciProviderName(AWS_CODEPIPELINE_PROVIDER_NAME)
+        .ciPipelineId(System.getenv(AWS_CODEPIPELINE_EXECUTION_ID))
+        .ciEnvVars(
+            AWS_CODEPIPELINE_EXECUTION_ID,
+            AWS_CODEPIPELINE_ACTION_EXECUTION_ID,
+            AWS_CODEPIPELINE_ARN)
+        .build();
+  }
+}

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/CIProviderInfoFactory.java

@@ -51,6 +51,9 @@ public CIProviderInfo createCIProviderInfo(Path currentPath) {
       return new CodefreshInfo();
     } else if (System.getenv(TeamcityInfo.TEAMCITY) != null) {
       return new TeamcityInfo();
+    } else if (System.getenv(AwsCodePipelineInfo.AWS_CODEPIPELINE) != null
+        && System.getenv(AwsCodePipelineInfo.AWS_CODEPIPELINE).startsWith("codepipeline")) {
+      return new AwsCodePipelineInfo();
     } else {
       return new UnknownCIInfo(targetFolder, currentPath);
     }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -14,7 +14,6 @@
 import java.lang.reflect.Method;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import javax.annotation.Nullable;
@@ -258,7 +257,7 @@ public void onTestIgnore(
       final @Nullable String testFramework,
       final @Nullable String testFrameworkVersion,
       final @Nullable String testParameters,
-      final @Nullable List<String> categories,
+      final @Nullable Collection<String> categories,
       final @Nullable Class<?> testClass,
       final @Nullable String testMethodName,
       final @Nullable Method testMethod,

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java

@@ -67,7 +67,7 @@ public boolean isShallow() throws IOException, TimeoutException, InterruptedExce
   }
 
   /**
-   * Returns the full symbolic name of the upstream (remote tracking) branch for the currently
+   * Returns the SHA of the head commit of the upstream (remote tracking) branch for the currently
    * checked-out local branch. If the local branch is not tracking any remote branches, a {@link
    * datadog.trace.civisibility.utils.ShellCommandExecutor.ShellCommandFailedException} exception
    * will be thrown.
@@ -79,15 +79,9 @@ public boolean isShallow() throws IOException, TimeoutException, InterruptedExce
    * @throws InterruptedException If current thread was interrupted while waiting for Git command to
    *     finish
    */
-  public String getUpstreamBranch() throws IOException, TimeoutException, InterruptedException {
+  public String getUpstreamBranchSha() throws IOException, TimeoutException, InterruptedException {
     return commandExecutor
-        .executeCommand(
-            IOUtils::readFully,
-            "git",
-            "rev-parse",
-            "--abbrev-ref",
-            "--symbolic-full-name",
-            "@{upstream}")
+        .executeCommand(IOUtils::readFully, "git", "rev-parse", "@{upstream}")
         .trim();
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataUploaderImpl.java

@@ -150,7 +150,7 @@ private void unshallowRepository() throws IOException, TimeoutException, Interru
     }
 
     try {
-      String upstreamBranch = gitClient.getUpstreamBranch();
+      String upstreamBranch = gitClient.getUpstreamBranchSha();
       gitClient.unshallow(upstreamBranch);
     } catch (ShellCommandExecutor.ShellCommandFailedException e) {
       LOGGER.debug(

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/AwsCodePipelineInfoTest.groovy

@@ -0,0 +1,30 @@
+package datadog.trace.civisibility.ci
+
+import java.nio.file.Path
+
+class AwsCodePipelineInfoTest extends CITagsProviderTest {
+
+  @Override
+  String getProviderName() {
+    return AwsCodePipelineInfo.AWS_CODEPIPELINE_PROVIDER_NAME
+  }
+
+  @Override
+  Map<String, String> buildRemoteGitInfoEmpty() {
+    final Map<String, String> map = new HashMap<>()
+    map.put(AwsCodePipelineInfo.AWS_CODEPIPELINE, "codepipeline")
+    return map
+  }
+
+  @Override
+  Map<String, String> buildRemoteGitInfoMismatchLocalGit() {
+    final Map<String, String> map = new HashMap<>()
+    map.put(AwsCodePipelineInfo.AWS_CODEPIPELINE, "codepipeline")
+    return map
+  }
+
+  @Override
+  Path getWorkspacePath() {
+    null
+  }
+}

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitClientTest.groovy

@@ -43,16 +43,16 @@ class GitClientTest extends Specification {
     shallow
   }
 
-  def "test get upstream branch"() {
+  def "test get upstream branch SHA"() {
     given:
     givenGitRepo("ci/git/shallow/git")
 
     when:
     def gitClient = givenGitClient()
-    def upstreamBranch = gitClient.getUpstreamBranch()
+    def upstreamBranch = gitClient.getUpstreamBranchSha()
 
     then:
-    upstreamBranch == "origin/master"
+    upstreamBranch == "98b944cc44f18bfb78e3021de2999cdcda8efdf6"
   }
 
   def "test unshallow: #remoteSha"() {