Merge with master

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 53d9b2c43876a39e5c0426f4df4dd5fd79d062e8
+default_system_tests_commit: &default_system_tests_commit 67fe30ac7504997685859d31feae1782f3527f39
 
 parameters:
   nightly:
@@ -816,7 +816,7 @@ jobs:
     parameters:
       weblog-variant:
         type: string
-    parallelism: 3
+    parallelism: 4
     steps:
       - setup_system_tests
 
@@ -838,11 +838,22 @@ jobs:
           no_output_timeout: 5m
           command: |
             cd system-tests
+            (
             echo "
             DEFAULT
             APM_TRACING_E2E
             APM_TRACING_E2E_SINGLE_SPAN
-            " | circleci tests split > scenarios.list
+            "
+            if ! [[ << parameters.weblog-variant >> =~ .*native ]]; then
+              echo "
+              APPSEC_BLOCKING
+              APPSEC_REQUEST_BLOCKING
+              APPSEC_RASP
+              APPSEC_RUNTIME_ACTIVATION
+              REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
+              "
+            fi
+            ) | circleci tests split > scenarios.list
             for scenario in $(<scenarios.list); do
               if [[ $scenario =~ .*_E2E.* ]]; then
                 export DD_SITE=datadoghq.com
@@ -860,6 +871,7 @@ jobs:
           command: |
             mkdir -p artifacts
             cd system-tests
+            shopt -s nullglob
             for log_dir in logs*; do
               tar -cvzf ../artifacts/${log_dir}_<< parameters.weblog-variant >>.tar.gz $log_dir
             done

.github/workflows/analyze-changes.yaml

@@ -16,18 +16,7 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
         with:
           submodules: 'recursive'
-      - name: Check code meets quality standards (production)
-        id: datadog-static-analysis
-        uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
-        with:
-          dd_app_key: ${{ secrets.DATADOG_APP_KEY_PROD }}
-          dd_api_key: ${{ secrets.DATADOG_API_KEY_PROD }}
-          dd_site: "datadoghq.com"
-          dd_service: "dd-trace-java"
-          dd_env: "ci"
-          cpu_count: 2
-          enable_performance_statistics: false
-      # Also run the static analysis on the staging environment to benefit from the new features not yet released
+      # Run the static analysis on the staging environment to benefit from the new features not yet released
       - name: Check code meets quality standards (staging)
         id: datadog-static-analysis-staging
         uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1

.github/workflows/check-pull-requests.yaml

@@ -27,7 +27,7 @@ jobs:
             const ignoreReleaseNotes = labels.filter(label => label == 'tag: no release notes').length > 0
             const hasTypeLabel = labels.filter(label => label.startsWith('type:')).length > 0
             const hasComponentLabel = labels.filter(label => label.startsWith('comp:')).length > 0
-            const hasInstrumentationLabel = labels.filter(label => label.startsWith('instr:')).length > 0
+            const hasInstrumentationLabel = labels.filter(label => label.startsWith('inst:')).length > 0
             const labelsCheckFailed = !ignoreReleaseNotes && (!hasTypeLabel || (!hasComponentLabel && !hasInstrumentationLabel));
             if (labelsCheckFailed) {
               core.setFailed('Please add at least one type, and one component or instrumentation label to the pull request.')

.github/workflows/tests/check-pull-requests/payload-pull-request-instrumentation.json

@@ -0,0 +1,15 @@
+{
+    "pull_request": {
+        "number": 7884,
+        "draft": false,
+        "labels": [
+            {
+                "name": "inst: java"
+            },
+            {
+                "name": "type: bug"
+            }
+        ],
+        "title": "Adding some new features"
+    }
+}

.github/workflows/tests/check-pull-requests/test-pull-request.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 source "$(dirname "$0")/../env.sh"
 testworkflow pull_request && \
+testworkflow pull_request instrumentation && \
 testworkflow pull_request draft && \
 testworkflow pull_request no-release-notes && \
 ! testworkflow pull_request missing-label && \

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java

@@ -1,17 +1,24 @@
 package datadog.trace.bootstrap.instrumentation.decorator;
 
 import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS;
+import static datadog.trace.api.gateway.Events.EVENTS;
 import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig;
 import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR;
 
+import datadog.appsec.api.blocking.BlockingException;
 import datadog.trace.api.Config;
 import datadog.trace.api.DDTags;
 import datadog.trace.api.InstrumenterConfig;
 import datadog.trace.api.ProductActivation;
+import datadog.trace.api.gateway.BlockResponseFunction;
+import datadog.trace.api.gateway.Flow;
+import datadog.trace.api.gateway.RequestContext;
+import datadog.trace.api.gateway.RequestContextSlot;
 import datadog.trace.api.iast.InstrumentationBridge;
 import datadog.trace.api.iast.sink.SsrfModule;
 import datadog.trace.api.naming.SpanNaming;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
+import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
 import datadog.trace.bootstrap.instrumentation.api.Tags;
 import datadog.trace.bootstrap.instrumentation.api.URIUtils;
@@ -21,6 +28,7 @@
 import java.util.BitSet;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.function.BiFunction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,6 +50,8 @@ public abstract class HttpClientDecorator<REQUEST, RESPONSE> extends UriBasedCli
 
   private static final boolean CLIENT_TAG_HEADERS = Config.get().isHttpClientTagHeaders();
 
+  private static final boolean APPSEC_RASP_ENABLED = Config.get().isAppSecRaspEnabled();
+
   protected abstract String method(REQUEST request);
 
   protected abstract URI url(REQUEST request) throws URISyntaxException;
@@ -68,9 +78,20 @@ protected boolean shouldSetResourceName() {
 
   public AgentSpan onRequest(final AgentSpan span, final REQUEST request) {
     if (request != null) {
+
       String method = method(request);
       span.setTag(Tags.HTTP_METHOD, method);
 
+      if (CLIENT_TAG_HEADERS) {
+        for (Map.Entry<String, String> headerTag :
+            traceConfig(span).getRequestHeaderTags().entrySet()) {
+          String headerValue = getRequestHeader(request, headerTag.getKey());
+          if (null != headerValue) {
+            span.setTag(headerTag.getValue(), headerValue);
+          }
+        }
+      }
+
       // Copy of HttpServerDecorator url handling
       try {
         final URI url = url(request);
@@ -86,24 +107,15 @@ public AgentSpan onRequest(final AgentSpan span, final REQUEST request) {
           if (shouldSetResourceName()) {
             HTTP_RESOURCE_DECORATOR.withClientPath(span, method, url.getPath());
           }
+          // SSRF exploit prevention check
+          onNetworkConnection(url.toString());
         } else if (shouldSetResourceName()) {
           span.setResourceName(DEFAULT_RESOURCE_NAME);
         }
       } catch (final Exception e) {
         log.debug("Error tagging url", e);
       }
-
       ssrfIastCheck(request);
-
-      if (CLIENT_TAG_HEADERS) {
-        for (Map.Entry<String, String> headerTag :
-            traceConfig(span).getRequestHeaderTags().entrySet()) {
-          String headerValue = getRequestHeader(request, headerTag.getKey());
-          if (null != headerValue) {
-            span.setTag(headerTag.getValue(), headerValue);
-          }
-        }
-      }
     }
     return span;
   }
@@ -175,6 +187,48 @@ public long getResponseContentLength(final RESPONSE response) {
     return 0;
   }
 
+  private void onNetworkConnection(final String networkConnection) {
+    if (!APPSEC_RASP_ENABLED) {
+      return;
+    }
+    if (networkConnection == null) {
+      return;
+    }
+    final BiFunction<RequestContext, String, Flow<Void>> networkConnectionCallback =
+        AgentTracer.get()
+            .getCallbackProvider(RequestContextSlot.APPSEC)
+            .getCallback(EVENTS.networkConnection());
+
+    if (networkConnectionCallback == null) {
+      return;
+    }
+
+    final AgentSpan span = AgentTracer.get().activeSpan();
+    if (span == null) {
+      return;
+    }
+
+    final RequestContext ctx = span.getRequestContext();
+    if (ctx == null) {
+      return;
+    }
+
+    Flow<Void> flow = networkConnectionCallback.apply(ctx, networkConnection);
+    Flow.Action action = flow.getAction();
+    if (action instanceof Flow.Action.RequestBlockingAction) {
+      BlockResponseFunction brf = ctx.getBlockResponseFunction();
+      if (brf != null) {
+        Flow.Action.RequestBlockingAction rba = (Flow.Action.RequestBlockingAction) action;
+        brf.tryCommitBlockingResponse(
+            ctx.getTraceSegment(),
+            rba.getStatusCode(),
+            rba.getBlockingContentType(),
+            rba.getExtraHeaders());
+      }
+      throw new BlockingException("Blocked request (for SSRF attempt)");
+    }
+  }
+
   /* This method must be overriden after making the proper propagations to the client before **/
   protected Object sourceUrl(REQUEST request) {
     return null;

dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecoratorTest.groovy

@@ -1,6 +1,12 @@
 package datadog.trace.bootstrap.instrumentation.decorator
 
 import datadog.trace.api.DDTags
+import datadog.trace.api.config.AppSecConfig
+import datadog.trace.api.gateway.CallbackProvider
+import static datadog.trace.api.gateway.Events.EVENTS
+import datadog.trace.api.gateway.RequestContext
+import datadog.trace.api.gateway.RequestContextSlot
+import datadog.trace.api.internal.TraceSegment
 import datadog.trace.api.iast.InstrumentationBridge
 import datadog.trace.api.iast.sink.SsrfModule
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan
@@ -12,11 +18,39 @@ import spock.lang.Shared
 import static datadog.trace.api.config.TraceInstrumentationConfig.HTTP_CLIENT_HOST_SPLIT_BY_DOMAIN
 import static datadog.trace.api.config.TraceInstrumentationConfig.HTTP_CLIENT_TAG_QUERY_STRING
 
+import java.util.function.BiFunction
+
 class HttpClientDecoratorTest extends ClientDecoratorTest {
 
   @Shared
   def testUrl = new URI("http://myhost:123/somepath")
 
+  @Shared
+  protected static final ORIGINAL_TRACER = AgentTracer.get()
+
+  protected traceSegment
+  protected reqCtx
+  protected span2
+  protected tracer
+
+  void setup() {
+    traceSegment = Stub(TraceSegment)
+    reqCtx = Stub(RequestContext) {
+      getTraceSegment() >> traceSegment
+    }
+    span2 = Stub(AgentSpan) {
+      getRequestContext() >> reqCtx
+    }
+    tracer = Stub(AgentTracer.TracerAPI) {
+      activeSpan() >> span2
+    }
+    AgentTracer.forceRegister(tracer)
+  }
+
+  void cleanup() {
+    AgentTracer.forceRegister(ORIGINAL_TRACER)
+  }
+
   def span = Mock(AgentSpan)
 
   def "test onRequest"() {
@@ -199,6 +233,26 @@ class HttpClientDecoratorTest extends ClientDecoratorTest {
     'false' | [method: "test-method", url: testUrl, path: '/somepath']
   }
 
+  void "test SSRF Exploit prevention onResponse"(){
+    setup:
+    injectSysConfig(AppSecConfig.APPSEC_ENABLED, 'true')
+    injectSysConfig(AppSecConfig.APPSEC_RASP_ENABLED, 'true')
+
+    final callbackProvider = Mock(CallbackProvider)
+    final listener = Mock(BiFunction)
+    tracer.getCallbackProvider(RequestContextSlot.APPSEC) >> callbackProvider
+
+    final decorator = newDecorator()
+    final req = [method: "GET", url: new URI("http://localhost:1234/somepath")]
+
+    when:
+    decorator.onRequest(span2,  req)
+
+    then:
+    1 * callbackProvider.getCallback(EVENTS.networkConnection()) >> listener
+    1 * listener.apply(reqCtx, _ as String)
+  }
+
   @Override
   def newDecorator(String serviceName = "test-service") {
     return new HttpClientDecorator<Map, Map>() {

dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecoratorTest.groovy

@@ -422,7 +422,8 @@ class HttpServerDecoratorTest extends ServerDecoratorTest {
     def ig = new InstrumentationGateway()
     def ss = ig.getSubscriptionService(RequestContextSlot.APPSEC)
     def cbpAppSec = ig.getCallbackProvider(RequestContextSlot.APPSEC)
-    def callbacks = new IGCallBacks(reqData)
+    def data = reqData ? new Object() : null
+    def callbacks = new IGCallBacks(data)
     if (reqStarted) {
       ss.registerCallback(EVENTS.requestStarted(), callbacks)
     }
@@ -434,7 +435,7 @@ class HttpServerDecoratorTest extends ServerDecoratorTest {
     }
     Map<String, String> headers = ["foo": "bar", "some": "thing", "another": "value"]
     def reqCtxt = Mock(RequestContext) {
-      getData(RequestContextSlot.APPSEC) >> reqData
+      getData(RequestContextSlot.APPSEC) >> data
     }
     def mSpan = Mock(AgentSpan) {
       getRequestContext() >> reqCtxt
@@ -461,13 +462,13 @@ class HttpServerDecoratorTest extends ServerDecoratorTest {
 
     where:
     // spotless:off
-    reqStarted | reqData      | reqHeader | reqHeaderDone | reqStartedCount | reqHeaderCount | reqHeaderDoneCount
-    false      | null         | false     | false         | 0               | 0              | 0
-    false      | new Object() | false     | false         | 0               | 0              | 0
-    true       | null         | false     | false         | 1               | 0              | 0
-    true       | new Object() | false     | false         | 1               | 0              | 0
-    true       | new Object() | true      | false         | 1               | 3              | 0
-    true       | new Object() | true      | true          | 1               | 3              | 1
+    reqStarted | reqData | reqHeader | reqHeaderDone | reqStartedCount | reqHeaderCount | reqHeaderDoneCount
+    false      | false   | false     | false         | 0               | 0              | 0
+    false      | true    | false     | false         | 0               | 0              | 0
+    true       | false   | false     | false         | 1               | 0              | 0
+    true       | true    | false     | false         | 1               | 0              | 0
+    true       | true    | true      | false         | 1               | 3              | 0
+    true       | true    | true      | true          | 1               | 3              | 1
     // spotless:on
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java

@@ -23,10 +23,10 @@
 import datadog.trace.civisibility.git.GitClientGitInfoBuilder;
 import datadog.trace.civisibility.git.tree.GitClient;
 import datadog.trace.civisibility.ipc.SignalClient;
-import datadog.trace.civisibility.source.BestEffortMethodLinesResolver;
-import datadog.trace.civisibility.source.ByteCodeMethodLinesResolver;
-import datadog.trace.civisibility.source.CompilerAidedMethodLinesResolver;
-import datadog.trace.civisibility.source.MethodLinesResolver;
+import datadog.trace.civisibility.source.BestEffortLinesResolver;
+import datadog.trace.civisibility.source.ByteCodeLinesResolver;
+import datadog.trace.civisibility.source.CompilerAidedLinesResolver;
+import datadog.trace.civisibility.source.LinesResolver;
 import datadog.trace.civisibility.source.index.*;
 import java.lang.reflect.Type;
 import java.net.InetSocketAddress;
@@ -63,7 +63,7 @@ public class CiVisibilityServices {
   final CIProviderInfoFactory ciProviderInfoFactory;
   final GitClient.Factory gitClientFactory;
   final GitInfoProvider gitInfoProvider;
-  final MethodLinesResolver methodLinesResolver;
+  final LinesResolver linesResolver;
   final RepoIndexProvider.Factory repoIndexProviderFactory;
   @Nullable final SignalClient.Factory signalClientFactory;
 
@@ -82,9 +82,8 @@ public class CiVisibilityServices {
 
     CiEnvironment environment = buildCiEnvironment(config, sco);
     this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
-    this.methodLinesResolver =
-        new BestEffortMethodLinesResolver(
-            new CompilerAidedMethodLinesResolver(), new ByteCodeMethodLinesResolver());
+    this.linesResolver =
+        new BestEffortLinesResolver(new CompilerAidedLinesResolver(), new ByteCodeLinesResolver());
 
     this.gitInfoProvider = gitInfoProvider;
     gitInfoProvider.registerGitInfoBuilder(new CIProviderGitInfoBuilder(config, environment));