Merge branch 'DataDog:master' into master

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit edfea31b7a9ceaed03b705de34a4e525853444c0
+default_system_tests_commit: &default_system_tests_commit ca7566bf7e66c0afdf29b0d3c07b2972e5869997
 
 parameters:
   nightly:
@@ -735,6 +735,7 @@ jobs:
           name: Generate muzzle dep report
           command: >-
             SKIP_BUILDSCAN="true"
+            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
             ./gradlew generateMuzzleReport muzzleInstrumentationReport
       - run:
           name: Collect Reports
@@ -1193,6 +1194,15 @@ build_test_jobs: &build_test_jobs
       stage: smoke
       cacheType: smoke
       testJvm: "graalvm17"
+  
+  - tests:
+      requires:
+        - ok_to_test
+      name: test_graalvm21_smoke
+      gradleTarget: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test"
+      stage: smoke
+      cacheType: smoke
+      testJvm: "graalvm21"
 
   - tests:
       requires:

.circleci/render_config.py

@@ -28,7 +28,7 @@
 }
 # Version to use for all the base Docker images, see
 # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build
-DOCKER_IMAGE_VERSION="v23.09"
+DOCKER_IMAGE_VERSION="v23.10"
 
 # Get labels from pull requests to override some defaults for jobs to run.
 # `run-tests: all` will run all tests.

.github/CODEOWNERS

@@ -19,6 +19,7 @@ dd-java-agent/instrumentation/cucumber/    @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/jacoco/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/junit-4.10/  @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/junit-5.3/   @DataDog/ci-app-libraries-java
+dd-java-agent/instrumentation/karate/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/testng/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/gradle/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/maven-3.2.1/ @DataDog/ci-app-libraries-java
@@ -36,6 +37,11 @@ dd-smoke-tests/debugger-integration-tests/ @DataDog/debugger-java
 dd-java-agent/agent-iast/              @DataDog/asm-java
 dd-java-agent/instrumentation/*iast*   @DataDog/asm-java
 dd-java-agent/instrumentation/*appsec* @DataDog/asm-java
+dd-java-agent/instrumentation/json/    @DataDog/asm-java
+dd-smoke-tests/iast-util/              @DataDog/asm-java
+dd-smoke-tests/spring-security/        @DataDog/asm-java
+dd-java-agent/instrumentation/commons-fileupload/    @DataDog/asm-java
+dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 **/appsec/                             @DataDog/asm-java
 **/iast/                               @DataDog/asm-java
 **/Iast*.java                          @DataDog/asm-java

.github/workflows/README.md

@@ -6,7 +6,7 @@ This lists and describes the repository GitHub actions.
 
 ### add-assets-to-release [🔗](add-assets-to-release.yaml)
 
-_Trigger:_ When a release is published
+_Trigger:_ When a release is published.
 
 _Actions:_
 * Ensure the release name is properly formatted (using `x.y.z` format),
@@ -16,15 +16,15 @@ _Recovery:_ Download artifacts and upload them manually to the release.
 
 ### add-milestone-to-pull-requests [🔗](add-milestone-to-pull-requests.yaml)
 
-_Trigger:_ When a PR to `master` is closed
+_Trigger:_ When a PR to `master` is closed.
 
 _Action:_ Get the last (by name) opened milestone and affect it to the closed pull request.
 
 _Recovery:_ Attach the milestone by hand to the PR.
 
 ### create-next-milestone [🔗](create-next-milestone.yaml)
 
-_Trigger:_ When closing a milestone
+_Trigger:_ When closing a milestone.
 
 _Action:_ Create a new milestone by incrementing minor version.
 
@@ -44,7 +44,7 @@ _Recovery:_ Manually trigger the action again on the relevant tag.
 
 ## increment-milestones-on-tag [🔗](increment-milestones-on-tag.yaml)
 
-_Trigger:_ When creating a tag
+_Trigger:_ When creating a tag.
 
 _Actions:_
 * Close the milestone related to the tag,
@@ -57,7 +57,7 @@ As there is no milestone for _patch_ releases, it won't close and create _patch_
 
 ## update-download-releases [🔗](update-download-releases.yaml)
 
-_Trigger:_ When a release is published
+_Trigger:_ When a release is published.
 
 _Action:_ Update the _download releases_ with the latest release artifact.
 
@@ -67,7 +67,7 @@ _Notes:_ _Download releases_ are special GitHub releases with fixed URL and tags
 
 ## update-issues-on-release [🔗](update-issues-on-release.yaml)
 
-_Trigger:_ When a release is published
+_Trigger:_ When a release is published.
 
 _Action:_
 * Find all issues related to the release by checking the related milestone,
@@ -78,15 +78,21 @@ _Recovery:_ Check at the milestone for the related issues and update them manual
 
 ## Code Quality and Security
 
+### comment-on-submodule-update [🔗](comment-on-submodule-update.yaml)
+
+_Trigger:_ When creating a PR commits to `master` or a `release/*` branch with a Git Submodule update.
+
+_Action:_ Notify the PR author through comments that about the Git Submodule update.
+
 ### codeql-analysis [🔗](codeql-analysis.yml)
 
-_Trigger:_ When pushing commits to `master` or any pull request to `master`
+_Trigger:_ When pushing commits to `master` or any pull request to `master`.
 
 _Action:_ Run GitHub CodeQL action and upload result to GitHub security tab.
 
 ### trivy-analysis [🔗](trivy-analysis.yml)
 
-_Trigger:_ When pushing commits to `master` or any pull request to `master`
+_Trigger:_ When pushing commits to `master` or any pull request to `master`.
 
 _Action:_ Run Trivy security scanner on built artifacts and upload result to GitHub security tab.
 
@@ -100,21 +106,21 @@ _Comment:_ To delete?
 
 ### lib-injection [🔗](lib-injection.yaml)
 
-_Trigger:_ When pushing commits to `master`, release branches or any PR targetting `master`, and when creating tags
+_Trigger:_ When pushing commits to `master`, release branches or any PR targetting `master`, and when creating tags.
 
 _Actions:_
 * Build and publish to GHCR a Docker image with the Java tracer agent,
 * Build lib-injection and run its system tests with the build Java agent.
 
 ### lib-injection-manual-release [🔗](lib-injection-manual-release.yaml)
 
-_Trigger:_ When manually triggered
+_Trigger:_ When manually triggered.
 
 _Action:_ Build and publish to GHCR a Docker image with the given Java tracer version.
 
 ### lib-injection-prune-registry [🔗](lib-injection-prune-registry.yaml)
 
-_Trigger:_ Every week or manually
+_Trigger:_ Every week or manually.
 
 _Action:_ Clean up old lib-injection Docker images from GHCR.
 

.github/workflows/comment-on-submodule-update.yaml

@@ -0,0 +1,29 @@
+name: Comment on Submodule Update
+
+on:
+  pull_request:
+    branches:
+        - 'master'
+        - 'release/**'
+    paths:
+      - 'dd-java-agent/agent-jmxfetch/integrations-core'
+
+jobs:
+  comment_on_submodule_update:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Post comment on submodule update
+        uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # 6.3.3
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Hi! 👋 Looks like you updated a Git Submodule.\n' +
+                'If this was not intentional please make sure to:\n\n' +
+                '* Update the submodule to the latest commit on the master branch using the `git submodule update` command,\n' +
+                '* Check you [properly set up your environment for contributing](https://github.com/DataDog/dd-trace-java/blob/master/CONTRIBUTING.md#git-submodule-setup).'
+            })

.gitlab-ci.yml

@@ -43,7 +43,7 @@ cache: &default_cache
 
 .gradle_build: &gradle_build
   <<: *common
-  image: ghcr.io/datadog/dd-trace-java-docker-build:base
+  image: ghcr.io/datadog/dd-trace-java-docker-build:v23.10-base
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
 
@@ -82,6 +82,12 @@ package:
   script:
     - ../.gitlab/build_java_package.sh
 
+package-arm:
+  extends: .package-arm
+  when: on_success # this can't use 'needs: [build]', since build is not available in the scheduled pipeline
+  script:
+    - ../.gitlab/build_java_package.sh
+
 .release-package:
   stage: deploy
   variables:

CONTRIBUTING.md

@@ -220,11 +220,11 @@ Suggested plugins and settings:
 
 To run tests on a different JVM than the one used for doing the build, you need two things:
 
-1) An environment variable pointing to the JVM to use on the form `JAVA_[JDKNAME]_HOME`, i.e. `JAVA_ZULU15_HOME`
+1) An environment variable pointing to the JVM to use on the form `JAVA_[JDKNAME]_HOME`, e.g. `JAVA_ZULU15_HOME`, `JAVA_GRAALVM17_HOME`
 
-2) A command line option to the gradle task on the form `-PtestJvm=[JDKNAME]`, i.e. `-PtestJvm=ZULU15`
+2) A command line option to the gradle task on the form `-PtestJvm=[JDKNAME]`, e.g. `-PtestJvm=ZULU15`, `-PtestJvm=GRAALVM17`
 
-Please note that the JDK name needs to end with the JDK version, i.e. `11`, `ZULU15`, `ORACLE8`, et.c.
+Please note that the JDK name needs to end with the JDK version, e.g. `11`, `ZULU15`, `ORACLE8`, `GRAALVM17`, etc.
 
 ## The APM Test Agent
 

buildSrc/src/main/groovy/InstrumentPlugin.groovy

@@ -116,7 +116,9 @@ abstract class InstrumentTask extends DefaultTask {
       parameters.buildStartedTime.set(invocationDetails.buildStartedTime)
       parameters.pluginClassPath.setFrom(project.configurations.findByName('instrumentPluginClasspath') ?: [])
       parameters.plugins.set(extension.plugins)
-      parameters.instrumentingClassPath.setFrom(project.configurations.compileClasspath.findAll {
+      def matcher = instrumentTask.name =~ /instrument([A-Z].+)Java/
+      def cfgName = matcher.matches() ? "${matcher.group(1).uncapitalize()}CompileClasspath" : 'compileClasspath'
+      parameters.instrumentingClassPath.setFrom(project.configurations[cfgName].findAll {
         it.name != 'previous-compilation-data.bin' && !it.name.endsWith(".gz")
       } + sourceDirectory + (extension.additionalClasspath[instrumentTask.name] ?: [])*.get())
       parameters.sourceDirectory.set(sourceDirectory.asFile)

communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java

@@ -47,6 +47,8 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
 
   public static final String DEBUGGER_ENDPOINT = "debugger/v1/input";
 
+  public static final String TELEMETRY_PROXY_ENDPOINT = "telemetry/proxy/";
+
   private static final long MIN_FEATURE_DISCOVERY_INTERVAL_MILLIS = 60 * 1000;
 
   private final OkHttpClient client;
@@ -58,6 +60,7 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
   private final boolean metricsEnabled;
   private final String[] dataStreamsEndpoints = {V01_DATASTREAMS_ENDPOINT};
   private final String[] evpProxyEndpoints = {V2_EVP_PROXY_ENDPOINT};
+  private final String[] telemetryProxyEndpoints = {TELEMETRY_PROXY_ENDPOINT};
 
   private volatile String traceEndpoint;
   private volatile String metricsEndpoint;
@@ -69,6 +72,7 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
   private volatile String debuggerEndpoint;
   private volatile String evpProxyEndpoint;
   private volatile String version;
+  private volatile String telemetryProxyEndpoint;
 
   private long lastTimeDiscovered;
 
@@ -100,6 +104,7 @@ private void reset() {
     evpProxyEndpoint = null;
     version = null;
     lastTimeDiscovered = 0;
+    telemetryProxyEndpoint = null;
   }
 
   /** Run feature discovery, unconditionally. */
@@ -162,14 +167,15 @@ private void doDiscovery() {
 
     if (log.isDebugEnabled()) {
       log.debug(
-          "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, evpProxyEndpoint={}",
+          "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, evpProxyEndpoint={}, telemetryProxyEndpoint={}",
           traceEndpoint,
           metricsEndpoint,
           supportsDropping,
           supportsLongRunning,
           dataStreamsEndpoint,
           configEndpoint,
-          evpProxyEndpoint);
+          evpProxyEndpoint,
+          telemetryProxyEndpoint);
     }
   }
 
@@ -247,6 +253,13 @@ private boolean processInfoResponse(String response) {
         }
       }
 
+      for (String endpoint : telemetryProxyEndpoints) {
+        if (endpoints.contains(endpoint) || endpoints.contains("/" + endpoint)) {
+          telemetryProxyEndpoint = endpoint;
+          break;
+        }
+      }
+
       supportsLongRunning = Boolean.TRUE.equals(map.getOrDefault("long_running_spans", false));
 
       if (metricsEnabled) {
@@ -352,4 +365,8 @@ public String state() {
   public boolean active() {
     return supportsMetrics() && supportsDropping;
   }
+
+  public boolean supportsTelemetryProxy() {
+    return telemetryProxyEndpoint != null;
+  }
 }

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -39,10 +39,15 @@ public void createRemaining(Config config) {
       String namedPipe = config.getAgentNamedPipe();
       okHttpClient =
           OkHttpUtils.buildHttpClient(
-              agentUrl,
-              unixDomainSocket,
-              namedPipe,
-              TimeUnit.SECONDS.toMillis(config.getAgentTimeout()));
+              agentUrl, unixDomainSocket, namedPipe, getHttpClientTimeout(config));
+    }
+  }
+
+  private static long getHttpClientTimeout(Config config) {
+    if (!config.isCiVisibilityEnabled() || !config.isCiVisibilityAgentlessEnabled()) {
+      return TimeUnit.SECONDS.toMillis(config.getAgentTimeout());
+    } else {
+      return config.getCiVisibilityBackendApiTimeoutMillis();
     }
   }
 

communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java

@@ -115,7 +115,6 @@ private void doConnect() {
         // when using UDS, set "entity-id" to "none" to avoid having the DogStatsD
         // server add origin tags (see https://github.com/DataDog/jmxfetch/pull/264)
         if (this.port == 0) {
-          clientBuilder.constantTags("dd.internal.card:none");
           clientBuilder.entityID("none");
         } else {
           clientBuilder.entityID(null);