Merge remote-tracking branch 'origin/master' into vandonr/wip

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit edfea31b7a9ceaed03b705de34a4e525853444c0
+default_system_tests_commit: &default_system_tests_commit 2487cea5160a398549743d2cfd927a863792e3bd
 
 parameters:
   nightly:

buildSrc/src/main/groovy/InstrumentPlugin.groovy

@@ -116,7 +116,9 @@ abstract class InstrumentTask extends DefaultTask {
       parameters.buildStartedTime.set(invocationDetails.buildStartedTime)
       parameters.pluginClassPath.setFrom(project.configurations.findByName('instrumentPluginClasspath') ?: [])
       parameters.plugins.set(extension.plugins)
-      parameters.instrumentingClassPath.setFrom(project.configurations.compileClasspath.findAll {
+      def matcher = instrumentTask.name =~ /instrument([A-Z].+)Java/
+      def cfgName = matcher.matches() ? "${matcher.group(1).uncapitalize()}CompileClasspath" : 'compileClasspath'
+      parameters.instrumentingClassPath.setFrom(project.configurations[cfgName].findAll {
         it.name != 'previous-compilation-data.bin' && !it.name.endsWith(".gz")
       } + sourceDirectory + (extension.additionalClasspath[instrumentTask.name] ?: [])*.get())
       parameters.sourceDirectory.set(sourceDirectory.asFile)

communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java

@@ -47,6 +47,8 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
 
   public static final String DEBUGGER_ENDPOINT = "debugger/v1/input";
 
+  public static final String TELEMETRY_PROXY_ENDPOINT = "telemetry/proxy/";
+
   private static final long MIN_FEATURE_DISCOVERY_INTERVAL_MILLIS = 60 * 1000;
 
   private final OkHttpClient client;
@@ -58,6 +60,7 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
   private final boolean metricsEnabled;
   private final String[] dataStreamsEndpoints = {V01_DATASTREAMS_ENDPOINT};
   private final String[] evpProxyEndpoints = {V2_EVP_PROXY_ENDPOINT};
+  private final String[] telemetryProxyEndpoints = {TELEMETRY_PROXY_ENDPOINT};
 
   private volatile String traceEndpoint;
   private volatile String metricsEndpoint;
@@ -69,6 +72,7 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
   private volatile String debuggerEndpoint;
   private volatile String evpProxyEndpoint;
   private volatile String version;
+  private volatile String telemetryProxyEndpoint;
 
   private long lastTimeDiscovered;
 
@@ -100,6 +104,7 @@ private void reset() {
     evpProxyEndpoint = null;
     version = null;
     lastTimeDiscovered = 0;
+    telemetryProxyEndpoint = null;
   }
 
   /** Run feature discovery, unconditionally. */
@@ -162,14 +167,15 @@ private void doDiscovery() {
 
     if (log.isDebugEnabled()) {
       log.debug(
-          "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, evpProxyEndpoint={}",
+          "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, evpProxyEndpoint={}, telemetryProxyEndpoint={}",
           traceEndpoint,
           metricsEndpoint,
           supportsDropping,
           supportsLongRunning,
           dataStreamsEndpoint,
           configEndpoint,
-          evpProxyEndpoint);
+          evpProxyEndpoint,
+          telemetryProxyEndpoint);
     }
   }
 
@@ -247,6 +253,13 @@ private boolean processInfoResponse(String response) {
         }
       }
 
+      for (String endpoint : telemetryProxyEndpoints) {
+        if (endpoints.contains(endpoint) || endpoints.contains("/" + endpoint)) {
+          telemetryProxyEndpoint = endpoint;
+          break;
+        }
+      }
+
       supportsLongRunning = Boolean.TRUE.equals(map.getOrDefault("long_running_spans", false));
 
       if (metricsEnabled) {
@@ -352,4 +365,8 @@ public String state() {
   public boolean active() {
     return supportsMetrics() && supportsDropping;
   }
+
+  public boolean supportsTelemetryProxy() {
+    return telemetryProxyEndpoint != null;
+  }
 }

communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy

@@ -15,7 +15,6 @@ import spock.lang.Shared
 
 import java.nio.file.Files
 import java.nio.file.Paths
-import java.util.concurrent.CountDownLatch
 
 import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V01_DATASTREAMS_ENDPOINT
 import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V6_METRICS_ENDPOINT
@@ -38,6 +37,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification {
   static final String INFO_WITHOUT_DATA_STREAMS_RESPONSE = loadJsonFile("agent-info-without-data-streams.json")
   static final String INFO_WITHOUT_DATA_STREAMS_STATE = Strings.sha256(INFO_WITHOUT_DATA_STREAMS_RESPONSE)
   static final String INFO_WITH_LONG_RUNNING_SPANS = loadJsonFile("agent-info-with-long-running-spans.json")
+  static final String INFO_WITH_TELEMETRY_PROXY_RESPONSE = loadJsonFile("agent-info-with-telemetry-proxy.json")
   static final String PROBE_STATE = "probestate"
 
   def "test parse /info response"() {
@@ -62,6 +62,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification {
     features.supportsEvpProxy()
     features.getVersion() == "0.99.0"
     !features.supportsLongRunning()
+    !features.supportsTelemetryProxy()
     0 * _
   }
 
@@ -89,6 +90,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification {
     features.supportsEvpProxy()
     features.getVersion() == "0.99.0"
     !features.supportsLongRunning()
+    !features.supportsTelemetryProxy()
     0 * _
   }
 
@@ -384,17 +386,22 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification {
     // but we don't permit dropping anyway
     !(features as DroppingPolicy).active()
     features.state() == INFO_WITHOUT_METRICS_STATE
+    !features.supportsTelemetryProxy()
     0 * _
   }
 
-  def countingNotFound(Request request, CountDownLatch latch) {
-    latch.countDown()
-    return notFound(request)
-  }
+  def "test parse /info response with telemetry proxy"() {
+    setup:
+    OkHttpClient client = Mock(OkHttpClient)
+    DDAgentFeaturesDiscovery features = new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, true, true)
 
-  def countingInfoResponse(Request request, String json, CountDownLatch latch) {
-    latch.countDown()
-    return infoResponse(request, json)
+    when: "/info available"
+    features.discover()
+
+    then:
+    1 * client.newCall(_) >> { Request request -> infoResponse(request, INFO_WITH_TELEMETRY_PROXY_RESPONSE) }
+    features.supportsTelemetryProxy()
+    0 * _
   }
 
   def infoResponse(Request request, String json) {

communication/src/test/resources/agent-features/agent-info-with-telemetry-proxy.json

@@ -0,0 +1,62 @@
+{
+  "version": "0.99.0",
+  "git_commit": "fab047e10",
+  "build_date": "2020-12-04 15:57:06.74187 +0200 EET m=+0.029001792",
+  "endpoints": [
+    "/v0.3/traces",
+    "/v0.3/services",
+    "/v0.4/traces",
+    "/v0.4/services",
+    "/v0.5/traces",
+    "/v0.6/stats",
+    "/profiling/v1/input",
+    "/telemetry/proxy/",
+    "/v0.1/pipeline_stats",
+    "/evp_proxy/v1/",
+    "/evp_proxy/v2/",
+    "/debugger/v1/input",
+    "/v0.7/config"
+  ],
+  "feature_flags": [
+    "feature_flag"
+  ],
+  "config": {
+    "default_env": "prod",
+    "bucket_interval": 1000000000,
+    "extra_aggregators": [
+      "agg:val"
+    ],
+    "extra_sample_rate": 2.4,
+    "target_tps": 11,
+    "max_eps": 12,
+    "receiver_port": 8111,
+    "receiver_socket": "/sock/path",
+    "connection_limit": 12,
+    "receiver_timeout": 100,
+    "max_request_bytes": 123,
+    "statsd_port": 123,
+    "max_memory": 1000000,
+    "max_cpu": 12345,
+    "analyzed_rate_by_service_legacy": {
+      "X": 1.2
+    },
+    "analyzed_spans_by_service": {
+      "X": {
+        "Y": 2.4
+      }
+    },
+    "obfuscation": {
+      "elastic_search": true,
+      "mongo": true,
+      "sql_exec_plan": true,
+      "sql_exec_plan_normalize": true,
+      "http": {
+        "remove_query_string": true,
+        "remove_path_digits": true
+      },
+      "remove_stack_traces": false,
+      "redis": true,
+      "memcached": false
+    }
+  }
+}

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -150,7 +150,13 @@ public static void start(final Instrumentation inst, final URL agentJarURL, Stri
     createAgentClassloader(agentJarURL);
 
     if (Platform.isNativeImageBuilder()) {
+      // these default services are not used during native-image builds
+      jmxFetchEnabled = false;
+      remoteConfigEnabled = false;
+      telemetryEnabled = false;
+      // apply trace instrumentation, but skip starting other services
       startDatadogAgent(inst);
+      StaticEventLogger.end("Agent.start");
       return;
     }
 

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java

@@ -143,7 +143,7 @@ public AgentSpan startSpan(
     }
     AgentPropagation.ContextVisitor<REQUEST_CARRIER> getter = getter();
     if (null != carrier && null != getter) {
-      tracer().getDataStreamsMonitoring().setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0);
+      tracer().getDataStreamsMonitoring().setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0, 0);
     }
     return span;
   }
@@ -250,7 +250,7 @@ public AgentSpan onRequest(
     }
 
     String inferredAddressStr = null;
-    if (clientIpResolverEnabled) {
+    if (clientIpResolverEnabled && context != null) {
       InetAddress inferredAddress = ClientIpAddressResolver.resolve(context, span);
       // the peer address should be used if:
       // 1. the headers yield nothing, regardless of whether it is public or not
@@ -269,6 +269,17 @@ public AgentSpan onRequest(
         inferredAddressStr = inferredAddress.getHostAddress();
         span.setTag(Tags.HTTP_CLIENT_IP, inferredAddressStr);
       }
+    } else if (clientIpResolverEnabled && span.getLocalRootSpan() != span) {
+      // in this case context == null
+      // If there is no context we can't do anything but use the peer addr.
+      // Additionally, context == null arises on subspans for which the resolution
+      // likely already happened on the top span, so we don't need to do the resolution
+      // again. Instead, copy from the top span, should it exist
+      AgentSpan localRootSpan = span.getLocalRootSpan();
+      Object clientIp = localRootSpan.getTag(Tags.HTTP_CLIENT_IP);
+      if (clientIp != null) {
+        span.setTag(Tags.HTTP_CLIENT_IP, clientIp);
+      }
     }
 
     if (peerIp != null) {

dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/WindowSampler.java

@@ -13,10 +13,14 @@ public class WindowSampler<E extends Event> {
 
   protected WindowSampler(
       Duration windowDuration, int samplesPerWindow, int lookback, Class<E> eventType) {
-    sampler = new AdaptiveSampler(windowDuration, samplesPerWindow, lookback, 16);
+    sampler = new AdaptiveSampler(windowDuration, samplesPerWindow, lookback, 16, false);
     sampleType = EventType.getEventType(eventType);
   }
 
+  public void start() {
+    sampler.start();
+  }
+
   public boolean sample() {
     return sampleType.isEnabled() && sampler.sample();
   }

dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionProfiling.java

@@ -42,13 +42,17 @@ private ExceptionProfiling(final Config config) {
     this.recordExceptionMessage = recordExceptionMessage;
   }
 
-  public ExceptionSampleEvent process(final Throwable t, final int stackDepth) {
+  public void start() {
+    sampler.start();
+  }
+
+  public ExceptionSampleEvent process(final Throwable t) {
     // always record the exception in histogram
     final boolean firstHit = histogram.record(t);
 
     final boolean sampled = sampler.sample();
     if (firstHit || sampled) {
-      return new ExceptionSampleEvent(t, stackDepth, sampled, firstHit);
+      return new ExceptionSampleEvent(t, sampled, firstHit);
     }
     return null;
   }

dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionSampleEvent.java

@@ -18,10 +18,6 @@ public class ExceptionSampleEvent extends Event implements ContextualEvent {
   @Label("Exception message")
   private final String message;
 
-  /** JFR may truncate the stack trace - so store original length as well. */
-  @Label("Exception stackdepth")
-  private final int stackDepth;
-
   @Label("Sampled")
   private final boolean sampled;
 
@@ -34,8 +30,7 @@ public class ExceptionSampleEvent extends Event implements ContextualEvent {
   @Label("Span Id")
   private long spanId;
 
-  public ExceptionSampleEvent(
-      Throwable e, final int stackDepth, boolean sampled, boolean firstOccurrence) {
+  public ExceptionSampleEvent(Throwable e, boolean sampled, boolean firstOccurrence) {
     /*
      * TODO: we should have some tests for this class.
      * Unfortunately at the moment this is not easily possible because we cannot build tests with groovy that
@@ -44,7 +39,6 @@ public ExceptionSampleEvent(
      */
     this.type = e.getClass().getName();
     this.message = getMessage(e);
-    this.stackDepth = stackDepth;
     this.sampled = sampled;
     this.firstOccurrence = firstOccurrence;
     captureContext();