diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 4a3d002d2b8..026026ea390 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core" profiling_modules: &profiling_modules "dd-java-agent/agent-profiling" -default_system_tests_commit: &default_system_tests_commit edfea31b7a9ceaed03b705de34a4e525853444c0 +default_system_tests_commit: &default_system_tests_commit 2487cea5160a398549743d2cfd927a863792e3bd parameters: nightly: diff --git a/buildSrc/src/main/groovy/InstrumentPlugin.groovy b/buildSrc/src/main/groovy/InstrumentPlugin.groovy index d19514e6543..34e41e48faa 100644 --- a/buildSrc/src/main/groovy/InstrumentPlugin.groovy +++ b/buildSrc/src/main/groovy/InstrumentPlugin.groovy @@ -116,7 +116,9 @@ abstract class InstrumentTask extends DefaultTask { parameters.buildStartedTime.set(invocationDetails.buildStartedTime) parameters.pluginClassPath.setFrom(project.configurations.findByName('instrumentPluginClasspath') ?: []) parameters.plugins.set(extension.plugins) - parameters.instrumentingClassPath.setFrom(project.configurations.compileClasspath.findAll { + def matcher = instrumentTask.name =~ /instrument([A-Z].+)Java/ + def cfgName = matcher.matches() ? "${matcher.group(1).uncapitalize()}CompileClasspath" : 'compileClasspath' + parameters.instrumentingClassPath.setFrom(project.configurations[cfgName].findAll { it.name != 'previous-compilation-data.bin' && !it.name.endsWith(".gz") } + sourceDirectory + (extension.additionalClasspath[instrumentTask.name] ?: [])*.get()) parameters.sourceDirectory.set(sourceDirectory.asFile) diff --git a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java index 1f1f73f1b7a..ae3c375875e 100644 --- a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java +++ b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java @@ -47,6 +47,8 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy { public static final String DEBUGGER_ENDPOINT = "debugger/v1/input"; + public static final String TELEMETRY_PROXY_ENDPOINT = "telemetry/proxy/"; + private static final long MIN_FEATURE_DISCOVERY_INTERVAL_MILLIS = 60 * 1000; private final OkHttpClient client; @@ -58,6 +60,7 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy { private final boolean metricsEnabled; private final String[] dataStreamsEndpoints = {V01_DATASTREAMS_ENDPOINT}; private final String[] evpProxyEndpoints = {V2_EVP_PROXY_ENDPOINT}; + private final String[] telemetryProxyEndpoints = {TELEMETRY_PROXY_ENDPOINT}; private volatile String traceEndpoint; private volatile String metricsEndpoint; @@ -69,6 +72,7 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy { private volatile String debuggerEndpoint; private volatile String evpProxyEndpoint; private volatile String version; + private volatile String telemetryProxyEndpoint; private long lastTimeDiscovered; @@ -100,6 +104,7 @@ private void reset() { evpProxyEndpoint = null; version = null; lastTimeDiscovered = 0; + telemetryProxyEndpoint = null; } /** Run feature discovery, unconditionally. */ @@ -162,14 +167,15 @@ private void doDiscovery() { if (log.isDebugEnabled()) { log.debug( - "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, evpProxyEndpoint={}", + "discovered traceEndpoint={}, metricsEndpoint={}, supportsDropping={}, supportsLongRunning={}, dataStreamsEndpoint={}, configEndpoint={}, evpProxyEndpoint={}, telemetryProxyEndpoint={}", traceEndpoint, metricsEndpoint, supportsDropping, supportsLongRunning, dataStreamsEndpoint, configEndpoint, - evpProxyEndpoint); + evpProxyEndpoint, + telemetryProxyEndpoint); } } @@ -247,6 +253,13 @@ private boolean processInfoResponse(String response) { } } + for (String endpoint : telemetryProxyEndpoints) { + if (endpoints.contains(endpoint) || endpoints.contains("/" + endpoint)) { + telemetryProxyEndpoint = endpoint; + break; + } + } + supportsLongRunning = Boolean.TRUE.equals(map.getOrDefault("long_running_spans", false)); if (metricsEnabled) { @@ -352,4 +365,8 @@ public String state() { public boolean active() { return supportsMetrics() && supportsDropping; } + + public boolean supportsTelemetryProxy() { + return telemetryProxyEndpoint != null; + } } diff --git a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy index af9b865d1b4..70b5519d355 100644 --- a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy +++ b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy @@ -15,7 +15,6 @@ import spock.lang.Shared import java.nio.file.Files import java.nio.file.Paths -import java.util.concurrent.CountDownLatch import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V01_DATASTREAMS_ENDPOINT import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V6_METRICS_ENDPOINT @@ -38,6 +37,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { static final String INFO_WITHOUT_DATA_STREAMS_RESPONSE = loadJsonFile("agent-info-without-data-streams.json") static final String INFO_WITHOUT_DATA_STREAMS_STATE = Strings.sha256(INFO_WITHOUT_DATA_STREAMS_RESPONSE) static final String INFO_WITH_LONG_RUNNING_SPANS = loadJsonFile("agent-info-with-long-running-spans.json") + static final String INFO_WITH_TELEMETRY_PROXY_RESPONSE = loadJsonFile("agent-info-with-telemetry-proxy.json") static final String PROBE_STATE = "probestate" def "test parse /info response"() { @@ -62,6 +62,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { features.supportsEvpProxy() features.getVersion() == "0.99.0" !features.supportsLongRunning() + !features.supportsTelemetryProxy() 0 * _ } @@ -89,6 +90,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { features.supportsEvpProxy() features.getVersion() == "0.99.0" !features.supportsLongRunning() + !features.supportsTelemetryProxy() 0 * _ } @@ -384,17 +386,22 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { // but we don't permit dropping anyway !(features as DroppingPolicy).active() features.state() == INFO_WITHOUT_METRICS_STATE + !features.supportsTelemetryProxy() 0 * _ } - def countingNotFound(Request request, CountDownLatch latch) { - latch.countDown() - return notFound(request) - } + def "test parse /info response with telemetry proxy"() { + setup: + OkHttpClient client = Mock(OkHttpClient) + DDAgentFeaturesDiscovery features = new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, true, true) - def countingInfoResponse(Request request, String json, CountDownLatch latch) { - latch.countDown() - return infoResponse(request, json) + when: "/info available" + features.discover() + + then: + 1 * client.newCall(_) >> { Request request -> infoResponse(request, INFO_WITH_TELEMETRY_PROXY_RESPONSE) } + features.supportsTelemetryProxy() + 0 * _ } def infoResponse(Request request, String json) { diff --git a/communication/src/test/resources/agent-features/agent-info-with-telemetry-proxy.json b/communication/src/test/resources/agent-features/agent-info-with-telemetry-proxy.json new file mode 100644 index 00000000000..55e493e8316 --- /dev/null +++ b/communication/src/test/resources/agent-features/agent-info-with-telemetry-proxy.json @@ -0,0 +1,62 @@ +{ + "version": "0.99.0", + "git_commit": "fab047e10", + "build_date": "2020-12-04 15:57:06.74187 +0200 EET m=+0.029001792", + "endpoints": [ + "/v0.3/traces", + "/v0.3/services", + "/v0.4/traces", + "/v0.4/services", + "/v0.5/traces", + "/v0.6/stats", + "/profiling/v1/input", + "/telemetry/proxy/", + "/v0.1/pipeline_stats", + "/evp_proxy/v1/", + "/evp_proxy/v2/", + "/debugger/v1/input", + "/v0.7/config" + ], + "feature_flags": [ + "feature_flag" + ], + "config": { + "default_env": "prod", + "bucket_interval": 1000000000, + "extra_aggregators": [ + "agg:val" + ], + "extra_sample_rate": 2.4, + "target_tps": 11, + "max_eps": 12, + "receiver_port": 8111, + "receiver_socket": "/sock/path", + "connection_limit": 12, + "receiver_timeout": 100, + "max_request_bytes": 123, + "statsd_port": 123, + "max_memory": 1000000, + "max_cpu": 12345, + "analyzed_rate_by_service_legacy": { + "X": 1.2 + }, + "analyzed_spans_by_service": { + "X": { + "Y": 2.4 + } + }, + "obfuscation": { + "elastic_search": true, + "mongo": true, + "sql_exec_plan": true, + "sql_exec_plan_normalize": true, + "http": { + "remove_query_string": true, + "remove_path_digits": true + }, + "remove_stack_traces": false, + "redis": true, + "memcached": false + } + } +} diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 90f1336a04c..a2343f12474 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -150,7 +150,13 @@ public static void start(final Instrumentation inst, final URL agentJarURL, Stri createAgentClassloader(agentJarURL); if (Platform.isNativeImageBuilder()) { + // these default services are not used during native-image builds + jmxFetchEnabled = false; + remoteConfigEnabled = false; + telemetryEnabled = false; + // apply trace instrumentation, but skip starting other services startDatadogAgent(inst); + StaticEventLogger.end("Agent.start"); return; } diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java index 777dfcacb65..334b2d665ba 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java @@ -143,7 +143,7 @@ public AgentSpan startSpan( } AgentPropagation.ContextVisitor getter = getter(); if (null != carrier && null != getter) { - tracer().getDataStreamsMonitoring().setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0); + tracer().getDataStreamsMonitoring().setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0, 0); } return span; } @@ -250,7 +250,7 @@ public AgentSpan onRequest( } String inferredAddressStr = null; - if (clientIpResolverEnabled) { + if (clientIpResolverEnabled && context != null) { InetAddress inferredAddress = ClientIpAddressResolver.resolve(context, span); // the peer address should be used if: // 1. the headers yield nothing, regardless of whether it is public or not @@ -269,6 +269,17 @@ public AgentSpan onRequest( inferredAddressStr = inferredAddress.getHostAddress(); span.setTag(Tags.HTTP_CLIENT_IP, inferredAddressStr); } + } else if (clientIpResolverEnabled && span.getLocalRootSpan() != span) { + // in this case context == null + // If there is no context we can't do anything but use the peer addr. + // Additionally, context == null arises on subspans for which the resolution + // likely already happened on the top span, so we don't need to do the resolution + // again. Instead, copy from the top span, should it exist + AgentSpan localRootSpan = span.getLocalRootSpan(); + Object clientIp = localRootSpan.getTag(Tags.HTTP_CLIENT_IP); + if (clientIp != null) { + span.setTag(Tags.HTTP_CLIENT_IP, clientIp); + } } if (peerIp != null) { diff --git a/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/WindowSampler.java b/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/WindowSampler.java index f48afc17856..f6489a29c2f 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/WindowSampler.java +++ b/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/WindowSampler.java @@ -13,10 +13,14 @@ public class WindowSampler { protected WindowSampler( Duration windowDuration, int samplesPerWindow, int lookback, Class eventType) { - sampler = new AdaptiveSampler(windowDuration, samplesPerWindow, lookback, 16); + sampler = new AdaptiveSampler(windowDuration, samplesPerWindow, lookback, 16, false); sampleType = EventType.getEventType(eventType); } + public void start() { + sampler.start(); + } + public boolean sample() { return sampleType.isEnabled() && sampler.sample(); } diff --git a/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionProfiling.java b/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionProfiling.java index 0cabe247278..d147017aedd 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionProfiling.java +++ b/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionProfiling.java @@ -42,13 +42,17 @@ private ExceptionProfiling(final Config config) { this.recordExceptionMessage = recordExceptionMessage; } - public ExceptionSampleEvent process(final Throwable t, final int stackDepth) { + public void start() { + sampler.start(); + } + + public ExceptionSampleEvent process(final Throwable t) { // always record the exception in histogram final boolean firstHit = histogram.record(t); final boolean sampled = sampler.sample(); if (firstHit || sampled) { - return new ExceptionSampleEvent(t, stackDepth, sampled, firstHit); + return new ExceptionSampleEvent(t, sampled, firstHit); } return null; } diff --git a/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionSampleEvent.java b/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionSampleEvent.java index 57de3176207..7dfbeaedb01 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionSampleEvent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java11/datadog/trace/bootstrap/instrumentation/jfr/exceptions/ExceptionSampleEvent.java @@ -18,10 +18,6 @@ public class ExceptionSampleEvent extends Event implements ContextualEvent { @Label("Exception message") private final String message; - /** JFR may truncate the stack trace - so store original length as well. */ - @Label("Exception stackdepth") - private final int stackDepth; - @Label("Sampled") private final boolean sampled; @@ -34,8 +30,7 @@ public class ExceptionSampleEvent extends Event implements ContextualEvent { @Label("Span Id") private long spanId; - public ExceptionSampleEvent( - Throwable e, final int stackDepth, boolean sampled, boolean firstOccurrence) { + public ExceptionSampleEvent(Throwable e, boolean sampled, boolean firstOccurrence) { /* * TODO: we should have some tests for this class. * Unfortunately at the moment this is not easily possible because we cannot build tests with groovy that @@ -44,7 +39,6 @@ public ExceptionSampleEvent( */ this.type = e.getClass().getName(); this.message = getMessage(e); - this.stackDepth = stackDepth; this.sampled = sampled; this.firstOccurrence = firstOccurrence; captureContext(); diff --git a/dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecoratorTest.groovy b/dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecoratorTest.groovy index 28379f84cb1..fcbba105a5a 100644 --- a/dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecoratorTest.groovy +++ b/dd-java-agent/agent-bootstrap/src/test/groovy/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecoratorTest.groovy @@ -23,6 +23,7 @@ import datadog.trace.core.datastreams.DataStreamsMonitoring import java.util.function.Function import java.util.function.Supplier +import static datadog.trace.api.config.TraceInstrumentationConfig.HTTP_SERVER_DECODED_RESOURCE_PRESERVE_SPACES import static datadog.trace.api.config.TraceInstrumentationConfig.HTTP_SERVER_RAW_QUERY_STRING import static datadog.trace.api.config.TraceInstrumentationConfig.HTTP_SERVER_RAW_RESOURCE import static datadog.trace.api.config.TraceInstrumentationConfig.HTTP_SERVER_TAG_QUERY_STRING @@ -63,6 +64,7 @@ class HttpServerDecoratorTest extends ServerDecoratorTest { } else { 1 * this.span.getRequestContext() } + _ * this.span.getLocalRootSpan() >> this.span 0 * _ where: @@ -102,6 +104,7 @@ class HttpServerDecoratorTest extends ServerDecoratorTest { 1 * this.span.setResourceName({ it as String == expectedPath }) } 1 * this.span.setTag(Tags.HTTP_METHOD, null) + _ * this.span.getLocalRootSpan() >> this.span 0 * _ where: @@ -141,18 +144,34 @@ class HttpServerDecoratorTest extends ServerDecoratorTest { 2 * this.span.getRequestContext() 1 * this.span.setResourceName({ it as String == expectedResource }, ResourceNamePriorities.HTTP_PATH_NORMALIZER) 1 * this.span.setTag(Tags.HTTP_METHOD, null) + _ * this.span.getLocalRootSpan() >> this.span 0 * _ where: rawQuery | rawResource | url | expectedUrl | expectedQuery | expectedResource - false | false | "http://host/p%20ath?query%3F?" | "http://host/p ath" | "query??" | "/path" + false | false | "http://host/p%20ath?query%3F?" | "http://host/p ath" | "query??" | "/p ath" false | true | "http://host/p%20ath?query%3F?" | "http://host/p%20ath" | "query??" | "/p%20ath" - true | false | "http://host/p%20ath?query%3F?" | "http://host/p ath" | "query%3F?" | "/path" + true | false | "http://host/p%20ath?query%3F?" | "http://host/p ath" | "query%3F?" | "/p ath" true | true | "http://host/p%20ath?query%3F?" | "http://host/p%20ath" | "query%3F?" | "/p%20ath" req = [url: url == null ? null : new URI(url)] } + void 'url handling without space preservation'() { + setup: + injectSysConfig(HTTP_SERVER_RAW_RESOURCE, 'false') + injectSysConfig(HTTP_SERVER_DECODED_RESOURCE_PRESERVE_SPACES, 'false') + def decorator = newDecorator() + + when: + decorator.onRequest(this.span, null, [url: new URI('http://host/p%20ath')], null) + + then: + 1 * this.span.setResourceName({ it as String == '/path' }, ResourceNamePriorities.HTTP_PATH_NORMALIZER) + _ * this.span.getLocalRootSpan() >> this.span + _ * _ + } + def "test onConnection"() { setup: def ctx = Mock(AgentSpan.Context.Extracted) @@ -269,6 +288,7 @@ class HttpServerDecoratorTest extends ServerDecoratorTest { when: decorator.onRequest(this.span, [peerIp: '4.4.4.4'], null, ctx) + _ * this.span.getLocalRootSpan() >> this.span then: 2 * ctx.getXForwardedFor() >> '2.3.4.5' 1 * this.span.setTag(Tags.HTTP_CLIENT_IP, '2.3.4.5') diff --git a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/CapturedContext.java b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/CapturedContext.java index 707877540f9..0bb90dc0305 100644 --- a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/CapturedContext.java +++ b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/CapturedContext.java @@ -327,26 +327,14 @@ public Status evaluate( ValueReferences.DURATION_EXTENSION_NAME, duration / 1_000_000.0); // convert to ms } this.thisClassName = thisClassName; - boolean shouldEvaluate = resolveEvaluateAt(probeImplementation, methodLocation); + boolean shouldEvaluate = + MethodLocation.isSame(methodLocation, probeImplementation.getEvaluateAt()); if (shouldEvaluate) { - probeImplementation.evaluate(this, status); + probeImplementation.evaluate(this, status, methodLocation); } return status; } - private static boolean resolveEvaluateAt( - ProbeImplementation probeImplementation, MethodLocation methodLocation) { - if (methodLocation == MethodLocation.DEFAULT) { - // line probe, no evaluation of probe's evaluateAt - return true; - } - MethodLocation localEvaluateAt = probeImplementation.getEvaluateAt(); - if (methodLocation == MethodLocation.ENTRY) { - return localEvaluateAt == MethodLocation.DEFAULT || localEvaluateAt == MethodLocation.ENTRY; - } - return localEvaluateAt == methodLocation; - } - public Status getStatus(String probeId) { Status result = statusByProbeId.get(probeId); if (result == null) { diff --git a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/DebuggerContext.java b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/DebuggerContext.java index e2009f95e5f..2ec97949aad 100644 --- a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/DebuggerContext.java +++ b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/DebuggerContext.java @@ -191,19 +191,9 @@ public static DebuggerSpan createSpan(String operationName, String[] tags) { * * @return true if can proceed to capture data */ - public static boolean isReadyToCapture(String... probeIds) { - // TODO provide overloaded version without string array + public static boolean isReadyToCapture(Class callingClass, String... probeIds) { try { - if (probeIds == null || probeIds.length == 0) { - return false; - } - boolean result = false; - for (String probeId : probeIds) { - // if all probes are rate limited, we don't capture - result |= ProbeRateLimiter.tryProbe(probeId); - } - result = result && checkAndSetInProbe(); - return result; + return checkAndSetInProbe(); } catch (Exception ex) { LOGGER.debug("Error in isReadyToCapture: ", ex); return false; diff --git a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/MethodLocation.java b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/MethodLocation.java index f07868286c4..7bceadf702d 100644 --- a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/MethodLocation.java +++ b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/MethodLocation.java @@ -3,5 +3,17 @@ public enum MethodLocation { DEFAULT, ENTRY, - EXIT + EXIT; + + public static boolean isSame(MethodLocation methodLocation, MethodLocation evaluateAt) { + if (methodLocation == MethodLocation.DEFAULT) { + // line probe, no evaluation of probe's evaluateAt + // MethodLocation.DEFAULT is used for line probe when evaluating the context + return true; + } + if (methodLocation == MethodLocation.ENTRY) { + return evaluateAt == MethodLocation.DEFAULT || evaluateAt == MethodLocation.ENTRY; + } + return methodLocation == evaluateAt; + } } diff --git a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeImplementation.java b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeImplementation.java index 80445b61fec..617b3aa6093 100644 --- a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeImplementation.java +++ b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeImplementation.java @@ -16,7 +16,8 @@ public interface ProbeImplementation { String getStrTags(); - void evaluate(CapturedContext context, CapturedContext.Status status); + void evaluate( + CapturedContext context, CapturedContext.Status status, MethodLocation methodLocation); void commit( CapturedContext entryContext, @@ -84,7 +85,8 @@ public String getStrTags() { } @Override - public void evaluate(CapturedContext context, CapturedContext.Status status) {} + public void evaluate( + CapturedContext context, CapturedContext.Status status, MethodLocation methodLocation) {} @Override public void commit( diff --git a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeRateLimiter.java b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeRateLimiter.java index 211a1ee34f1..e7c7aa22f8d 100644 --- a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeRateLimiter.java +++ b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/ProbeRateLimiter.java @@ -7,6 +7,7 @@ import java.time.temporal.ChronoUnit; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.DoubleFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +24,7 @@ public class ProbeRateLimiter { new ConcurrentHashMap<>(); private static Sampler GLOBAL_SNAPSHOT_SAMPLER = createSampler(DEFAULT_GLOBAL_SNAPSHOT_RATE); private static Sampler GLOBAL_LOG_SAMPLER = createSampler(DEFAULT_GLOBAL_LOG_RATE); + private static DoubleFunction samplerSupplier = ProbeRateLimiter::createSampler; public static boolean tryProbe(String probeId) { RateLimitInfo rateLimitInfo = @@ -37,19 +39,19 @@ public static boolean tryProbe(String probeId) { private static RateLimitInfo getDefaultRateLimitInfo(String probeId) { LOGGER.debug("Setting sampling with default snapshot rate for probeId={}", probeId); - return new RateLimitInfo(createSampler(DEFAULT_SNAPSHOT_RATE), true); + return new RateLimitInfo(samplerSupplier.apply(DEFAULT_SNAPSHOT_RATE), true); } public static void setRate(String probeId, double rate, boolean isCaptureSnapshot) { - PROBE_SAMPLERS.put(probeId, new RateLimitInfo(createSampler(rate), isCaptureSnapshot)); + PROBE_SAMPLERS.put(probeId, new RateLimitInfo(samplerSupplier.apply(rate), isCaptureSnapshot)); } public static void setGlobalSnapshotRate(double rate) { - GLOBAL_SNAPSHOT_SAMPLER = createSampler(rate); + GLOBAL_SNAPSHOT_SAMPLER = samplerSupplier.apply(rate); } public static void setGlobalLogRate(double rate) { - GLOBAL_LOG_SAMPLER = createSampler(rate); + GLOBAL_LOG_SAMPLER = samplerSupplier.apply(rate); } public static void resetRate(String probeId) { @@ -65,15 +67,20 @@ public static void resetAll() { resetGlobalRate(); } + public static void setSamplerSupplier(DoubleFunction samplerSupplier) { + ProbeRateLimiter.samplerSupplier = + samplerSupplier != null ? samplerSupplier : ProbeRateLimiter::createSampler; + } + private static Sampler createSampler(double rate) { if (rate < 0) { return new ConstantSampler(true); } if (rate < 1) { int intRate = (int) Math.round(rate * 10); - return new AdaptiveSampler(TEN_SECONDS_WINDOW, intRate, 180, 16); + return new AdaptiveSampler(TEN_SECONDS_WINDOW, intRate, 180, 16, true); } - return new AdaptiveSampler(ONE_SECOND_WINDOW, (int) Math.round(rate), 180, 16); + return new AdaptiveSampler(ONE_SECOND_WINDOW, (int) Math.round(rate), 180, 16, true); } private static class RateLimitInfo { diff --git a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/util/Redaction.java b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/util/Redaction.java index 2d746395f68..41383d3c50a 100644 --- a/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/util/Redaction.java +++ b/dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/util/Redaction.java @@ -13,7 +13,7 @@ public class Redaction { // Need to be a unique instance (new String) for reference equality (==) and // avoid internalization (intern) by the JVM because it's a string constant - public static final String REDACTED_VALUE = new String("REDACTED".toCharArray()); + public static final String REDACTED_VALUE = new String("redacted".toCharArray()); private static final Pattern COMMA_PATTERN = Pattern.compile(","); private static final List PREDEFINED_KEYWORDS = diff --git a/dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/RedactedException.java b/dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/RedactedException.java new file mode 100644 index 00000000000..a2ec6cdb02d --- /dev/null +++ b/dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/RedactedException.java @@ -0,0 +1,7 @@ +package com.datadog.debugger.el; + +public class RedactedException extends EvaluationException { + public RedactedException(String message, String expr) { + super(message, expr); + } +} diff --git a/dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/ExpressionHelper.java b/dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/ExpressionHelper.java index e77f17286dd..43d5930dcb6 100644 --- a/dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/ExpressionHelper.java +++ b/dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/ExpressionHelper.java @@ -1,13 +1,13 @@ package com.datadog.debugger.el.expressions; -import com.datadog.debugger.el.EvaluationException; import com.datadog.debugger.el.Expression; import com.datadog.debugger.el.PrettyPrintVisitor; +import com.datadog.debugger.el.RedactedException; public class ExpressionHelper { public static void throwRedactedException(Expression expr) { String strExpr = PrettyPrintVisitor.print(expr); - throw new EvaluationException( + throw new RedactedException( "Could not evaluate the expression because '" + strExpr + "' was redacted", strExpr); } } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ConfigurationUpdater.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ConfigurationUpdater.java index f45c17584b4..533d78e2a83 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ConfigurationUpdater.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ConfigurationUpdater.java @@ -258,11 +258,6 @@ private void applyRateLimiter(ConfigurationComparer changes) { : getDefaultRateLimitPerProbe(probe), probe.isCaptureSnapshot()); } - if (addedDefinitions instanceof SpanDecorationProbe) { - // Span decoration probes use the same instrumentation as log probes, but we don't want - // to sample here. - ProbeRateLimiter.setRate(addedDefinitions.getId(), -1, false); - } } // remove rate for all removed probes for (ProbeDefinition removedDefinition : changes.getRemovedDefinitions()) { diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java index 5c864cbf33d..7d4a888805d 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java @@ -4,6 +4,7 @@ import com.datadog.debugger.sink.DebuggerSink; import com.datadog.debugger.sink.Sink; +import com.datadog.debugger.symbol.SymbolExtractionTransformer; import com.datadog.debugger.uploader.BatchUploader; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; @@ -91,6 +92,10 @@ public static synchronized void run( } else { log.debug("No configuration poller available from SharedCommunicationObjects"); } + if (config.isDebuggerSymbolEnabled() && config.isDebuggerSymbolForceUpload()) { + instrumentation.addTransformer( + new SymbolExtractionTransformer(debuggerSink.getSymbolSink(), config)); + } } private static void setupSourceFileTracking( diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/LogMessageTemplateBuilder.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/LogMessageTemplateBuilder.java index 4554630e8eb..6b314725870 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/LogMessageTemplateBuilder.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/LogMessageTemplateBuilder.java @@ -3,11 +3,13 @@ import static com.datadog.debugger.util.ValueScriptHelper.serializeValue; import com.datadog.debugger.el.EvaluationException; +import com.datadog.debugger.el.RedactedException; import com.datadog.debugger.el.Value; import com.datadog.debugger.el.ValueScript; import com.datadog.debugger.probe.LogProbe; import datadog.trace.bootstrap.debugger.CapturedContext; import datadog.trace.bootstrap.debugger.EvaluationError; +import datadog.trace.bootstrap.debugger.util.Redaction; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,7 +51,9 @@ public String evaluate(CapturedContext context, LogProbe.LogStatus status) { LOGGER.debug("Evaluation error: ", ex); status.addError(new EvaluationError(ex.getExpr(), ex.getMessage())); status.setLogTemplateErrors(true); - sb.append("{").append(ex.getMessage()).append("}"); + String msg = + ex instanceof RedactedException ? Redaction.REDACTED_VALUE : ex.getMessage(); + sb.append("{").append(msg).append("}"); } } } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/ASMHelper.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/ASMHelper.java index 3edb9a464f0..eb32212e0fc 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/ASMHelper.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/ASMHelper.java @@ -9,6 +9,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; import org.objectweb.asm.Opcodes; @@ -20,6 +21,7 @@ import org.objectweb.asm.tree.InsnList; import org.objectweb.asm.tree.InsnNode; import org.objectweb.asm.tree.LdcInsnNode; +import org.objectweb.asm.tree.LocalVariableNode; import org.objectweb.asm.tree.MethodInsnNode; import org.objectweb.asm.tree.TypeInsnNode; @@ -203,6 +205,43 @@ private static String getReflectiveMethodName(int sort) { } } + public static List sortLocalVariables(List localVariables) { + List sortedLocalVars = new ArrayList<>(localVariables); + sortedLocalVars.sort(Comparator.comparingInt(o -> o.index)); + return sortedLocalVars; + } + + public static LocalVariableNode[] createLocalVarNodes(List sortedLocalVars) { + int maxIndex = sortedLocalVars.get(sortedLocalVars.size() - 1).index; + LocalVariableNode[] localVars = new LocalVariableNode[maxIndex + 1]; + for (LocalVariableNode localVariableNode : sortedLocalVars) { + localVars[localVariableNode.index] = localVariableNode; + } + return localVars; + } + + public static void adjustLocalVarsBasedOnArgs( + boolean isStatic, + LocalVariableNode[] localVars, + org.objectweb.asm.Type[] argTypes, + List sortedLocalVars) { + // assume that first local variables matches method arguments + // as stated into the JVM spec: + // https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-2.html#jvms-2.6.1 + // so we reassigned local var in arg slots if they are empty + if (argTypes.length < localVars.length) { + int slot = isStatic ? 0 : 1; + int localVarTableIdx = slot; + for (org.objectweb.asm.Type t : argTypes) { + if (localVars[slot] == null && localVarTableIdx < sortedLocalVars.size()) { + localVars[slot] = sortedLocalVars.get(localVarTableIdx); + } + slot += t.getSize(); + localVarTableIdx++; + } + } + } + /** Wraps ASM's {@link org.objectweb.asm.Type} with associated generic types */ public static class Type { private final org.objectweb.asm.Type mainType; diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/CapturedContextInstrumentor.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/CapturedContextInstrumentor.java index 73ef9efb8fa..f728e14eb30 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/CapturedContextInstrumentor.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/CapturedContextInstrumentor.java @@ -127,6 +127,8 @@ private boolean addLineCaptures(LineMap lineMap) { } if (beforeLabel != null) { InsnList insnList = new InsnList(); + ldc(insnList, Type.getObjectType(classNode.name)); + // stack [class, array] pushProbesIds(insnList); // stack [array] invokeStatic( @@ -134,6 +136,7 @@ private boolean addLineCaptures(LineMap lineMap) { DEBUGGER_CONTEXT_TYPE, "isReadyToCapture", Type.BOOLEAN_TYPE, + CLASS_TYPE, STRING_ARRAY_TYPE); // stack [boolean] LabelNode targetNode = new LabelNode(); @@ -314,10 +317,17 @@ private void instrumentMethodEnter() { methodNode.instructions.insert(methodEnterLabel, insnList); return; } + ldc(insnList, Type.getObjectType(classNode.name)); + // stack [class] pushProbesIds(insnList); - // stack [array] + // stack [class, array] invokeStatic( - insnList, DEBUGGER_CONTEXT_TYPE, "isReadyToCapture", Type.BOOLEAN_TYPE, STRING_ARRAY_TYPE); + insnList, + DEBUGGER_CONTEXT_TYPE, + "isReadyToCapture", + Type.BOOLEAN_TYPE, + CLASS_TYPE, + STRING_ARRAY_TYPE); // stack [boolean] LabelNode targetNode = new LabelNode(); LabelNode gotoNode = new LabelNode(); diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/Instrumentor.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/Instrumentor.java index 91799dfeff5..cda1012b299 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/Instrumentor.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/instrumentation/Instrumentor.java @@ -1,14 +1,15 @@ package com.datadog.debugger.instrumentation; +import static com.datadog.debugger.instrumentation.ASMHelper.adjustLocalVarsBasedOnArgs; +import static com.datadog.debugger.instrumentation.ASMHelper.createLocalVarNodes; import static com.datadog.debugger.instrumentation.ASMHelper.ldc; +import static com.datadog.debugger.instrumentation.ASMHelper.sortLocalVariables; import static com.datadog.debugger.instrumentation.Types.STRING_TYPE; import com.datadog.debugger.instrumentation.DiagnosticMessage.Kind; import com.datadog.debugger.probe.ProbeDefinition; import com.datadog.debugger.probe.Where; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.List; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; @@ -75,29 +76,10 @@ private LocalVariableNode[] extractLocalVariables(Type[] argTypes) { if (methodNode.localVariables == null || methodNode.localVariables.isEmpty()) { return new LocalVariableNode[0]; } - List sortedLocalVars = new ArrayList<>(methodNode.localVariables); - sortedLocalVars.sort(Comparator.comparingInt(o -> o.index)); - int maxIndex = sortedLocalVars.get(sortedLocalVars.size() - 1).index; - LocalVariableNode[] localVars = new LocalVariableNode[maxIndex + 1]; + List sortedLocalVars = sortLocalVariables(methodNode.localVariables); + LocalVariableNode[] localVars = createLocalVarNodes(sortedLocalVars); + adjustLocalVarsBasedOnArgs(isStatic, localVars, argTypes, sortedLocalVars); localVarBaseOffset = sortedLocalVars.get(0).index; - for (LocalVariableNode localVariableNode : sortedLocalVars) { - localVars[localVariableNode.index] = localVariableNode; - } - // assume that first local variables matches method arguments - // as stated into the JVM spec: - // https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-2.html#jvms-2.6.1 - // so we reassigned local var in arg slots if they are empty - if (argTypes.length < localVars.length) { - int slot = isStatic ? 0 : 1; - int localVarTableIdx = slot; - for (Type t : argTypes) { - if (localVars[slot] == null && localVarTableIdx < sortedLocalVars.size()) { - localVars[slot] = sortedLocalVars.get(localVarTableIdx); - } - slot += t.getSize(); - localVarTableIdx++; - } - } return localVars; } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java index 48628f10550..acb71fa7f50 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java @@ -363,11 +363,17 @@ public InstrumentationResult.Status instrument( } @Override - public void evaluate(CapturedContext context, CapturedContext.Status status) { + public void evaluate( + CapturedContext context, CapturedContext.Status status, MethodLocation methodLocation) { if (!(status instanceof LogStatus)) { throw new IllegalStateException("Invalid status: " + status.getClass()); } + LogStatus logStatus = (LogStatus) status; + if (!hasCondition()) { + // sample when no condition associated + sample(logStatus, methodLocation); + } logStatus.setCondition(evaluateCondition(context, logStatus)); CapturedContext.CapturedThrowable throwable = context.getThrowable(); if (logStatus.hasConditionErrors() && throwable != null) { @@ -375,12 +381,29 @@ public void evaluate(CapturedContext context, CapturedContext.Status status) { new EvaluationError( "uncaught exception", throwable.getType() + ": " + throwable.getMessage())); } - if (logStatus.getCondition()) { + if (hasCondition() && logStatus.getCondition()) { + // sample if probe has condition and condition is true + sample(logStatus, methodLocation); + } + if (logStatus.isSampled() && logStatus.getCondition()) { LogMessageTemplateBuilder logMessageBuilder = new LogMessageTemplateBuilder(segments); logStatus.setMessage(logMessageBuilder.evaluate(context, logStatus)); } } + private void sample(LogStatus logStatus, MethodLocation methodLocation) { + // sample only once and when we need to evaluate + if (!MethodLocation.isSame(methodLocation, evaluateAt)) { + return; + } + boolean sampled = ProbeRateLimiter.tryProbe(id); + logStatus.setSampled(sampled); + if (!sampled) { + LOGGER.debug("{} not sampled!", id); + DebuggerAgent.getSink().skipSnapshot(id, DebuggerContext.SkipCause.RATE); + } + } + private boolean evaluateCondition(CapturedContext capture, LogStatus status) { if (probeCondition == null) { return true; @@ -430,13 +453,6 @@ public void commit( int maxDepth = capture != null ? capture.maxReferenceDepth : -1; Snapshot snapshot = new Snapshot(Thread.currentThread(), this, maxDepth); if (entryStatus.shouldSend() && exitStatus.shouldSend()) { - // only rate limit if a condition is defined - if (probeCondition != null) { - if (!ProbeRateLimiter.tryProbe(id)) { - sink.skipSnapshot(id, DebuggerContext.SkipCause.RATE); - return; - } - } snapshot.setTraceId(traceId); snapshot.setSpanId(spanId); if (isCaptureSnapshot()) { @@ -506,13 +522,6 @@ public void commit(CapturedContext lineContext, int line) { Snapshot snapshot = new Snapshot(Thread.currentThread(), this, maxDepth); boolean shouldCommit = false; if (status.shouldSend()) { - // only rate limit if a condition is defined - if (probeCondition != null) { - if (!ProbeRateLimiter.tryProbe(id)) { - sink.skipSnapshot(id, DebuggerContext.SkipCause.RATE); - return; - } - } snapshot.setTraceId(lineContext.getTraceId()); snapshot.setSpanId(lineContext.getSpanId()); if (isCaptureSnapshot()) { @@ -554,6 +563,7 @@ public static class LogStatus extends CapturedContext.Status { private boolean condition = true; private boolean hasLogTemplateErrors; private boolean hasConditionErrors; + private boolean sampled = true; private String message; public LogStatus(ProbeImplementation probeImplementation) { @@ -567,7 +577,7 @@ private LogStatus(ProbeImplementation probeImplementation, boolean condition) { @Override public boolean shouldFreezeContext() { - return probeImplementation.isCaptureSnapshot() && shouldSend(); + return sampled && probeImplementation.isCaptureSnapshot() && shouldSend(); } @Override @@ -576,7 +586,7 @@ public boolean isCapturing() { } public boolean shouldSend() { - return condition && !hasConditionErrors; + return sampled && condition && !hasConditionErrors; } public boolean shouldReportError() { @@ -614,6 +624,14 @@ public void setMessage(String message) { public String getMessage() { return message; } + + public void setSampled(boolean sampled) { + this.sampled = sampled; + } + + public boolean isSampled() { + return sampled; + } } @Generated diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/ProbeDefinition.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/ProbeDefinition.java index a02b1e95646..4de096f0619 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/ProbeDefinition.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/ProbeDefinition.java @@ -143,7 +143,8 @@ public ProbeLocation getLocation() { } @Override - public void evaluate(CapturedContext context, CapturedContext.Status status) {} + public void evaluate( + CapturedContext context, CapturedContext.Status status, MethodLocation methodLocation) {} @Override public void commit( diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/SpanDecorationProbe.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/SpanDecorationProbe.java index 9bd2a410dfe..5de07fc1a7b 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/SpanDecorationProbe.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/SpanDecorationProbe.java @@ -145,7 +145,8 @@ public InstrumentationResult.Status instrument( } @Override - public void evaluate(CapturedContext context, CapturedContext.Status status) { + public void evaluate( + CapturedContext context, CapturedContext.Status status, MethodLocation methodLocation) { for (Decoration decoration : decorations) { if (decoration.when != null) { try { diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/DebuggerSink.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/DebuggerSink.java index 8203869d9a0..6167e2de60d 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/DebuggerSink.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/DebuggerSink.java @@ -30,6 +30,7 @@ public class DebuggerSink implements Sink { private final ProbeStatusSink probeStatusSink; private final SnapshotSink snapshotSink; + private final SymbolSink symbolSink; private final DebuggerMetrics debuggerMetrics; private final BatchUploader batchUploader; private final String tags; @@ -42,10 +43,11 @@ public class DebuggerSink implements Sink { public DebuggerSink(Config config) { this( config, - new BatchUploader(config), + new BatchUploader(config, config.getFinalDebuggerSnapshotUrl()), DebuggerMetrics.getInstance(config), new ProbeStatusSink(config), - new SnapshotSink(config)); + new SnapshotSink(config), + new SymbolSink(config)); } DebuggerSink(Config config, BatchUploader batchUploader) { @@ -54,16 +56,18 @@ public DebuggerSink(Config config) { batchUploader, DebuggerMetrics.getInstance(config), new ProbeStatusSink(config), - new SnapshotSink(config)); + new SnapshotSink(config), + new SymbolSink(config)); } public DebuggerSink(Config config, ProbeStatusSink probeStatusSink) { this( config, - new BatchUploader(config), + new BatchUploader(config, config.getFinalDebuggerSnapshotUrl()), DebuggerMetrics.getInstance(config), probeStatusSink, - new SnapshotSink(config)); + new SnapshotSink(config), + new SymbolSink(config)); } DebuggerSink(Config config, BatchUploader batchUploader, DebuggerMetrics debuggerMetrics) { @@ -72,7 +76,8 @@ public DebuggerSink(Config config, ProbeStatusSink probeStatusSink) { batchUploader, debuggerMetrics, new ProbeStatusSink(config), - new SnapshotSink(config)); + new SnapshotSink(config), + new SymbolSink(config)); } public DebuggerSink( @@ -80,12 +85,14 @@ public DebuggerSink( BatchUploader batchUploader, DebuggerMetrics debuggerMetrics, ProbeStatusSink probeStatusSink, - SnapshotSink snapshotSink) { + SnapshotSink snapshotSink, + SymbolSink symbolSink) { this.batchUploader = batchUploader; tags = getDefaultTagsMergedWithGlobalTags(config); this.debuggerMetrics = debuggerMetrics; this.probeStatusSink = probeStatusSink; this.snapshotSink = snapshotSink; + this.symbolSink = symbolSink; this.uploadFlushInterval = config.getDebuggerUploadFlushInterval(); } @@ -136,6 +143,10 @@ public BatchUploader getSnapshotUploader() { return batchUploader; } + public SymbolSink getSymbolSink() { + return symbolSink; + } + @Override public void addSnapshot(Snapshot snapshot) { boolean added = snapshotSink.offer(snapshot); @@ -160,15 +171,16 @@ private void reschedule() { // visible for testing void flush(DebuggerSink ignored) { + symbolSink.flush(); List diagnostics = probeStatusSink.getSerializedDiagnostics(); List snapshots = snapshotSink.getSerializedSnapshots(); if (snapshots.size() + diagnostics.size() == 0) { return; } - if (snapshots.size() >= 1) { + if (snapshots.size() > 0) { uploadPayloads(snapshots); } - if (diagnostics.size() >= 1) { + if (diagnostics.size() > 0) { uploadPayloads(diagnostics); } } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SymbolSink.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SymbolSink.java new file mode 100644 index 00000000000..1ff688a35ad --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SymbolSink.java @@ -0,0 +1,85 @@ +package com.datadog.debugger.sink; + +import com.datadog.debugger.symbol.Scope; +import com.datadog.debugger.symbol.ServiceVersion; +import com.datadog.debugger.uploader.BatchUploader; +import com.datadog.debugger.util.ExceptionHelper; +import com.datadog.debugger.util.MoshiHelper; +import com.squareup.moshi.JsonAdapter; +import datadog.trace.api.Config; +import datadog.trace.util.TagsHelper; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SymbolSink { + + private static final Logger LOGGER = LoggerFactory.getLogger(SymbolSink.class); + private static final int CAPACITY = 1024; + private static final JsonAdapter SERVICE_VERSION_ADAPTER = + MoshiHelper.createMoshiSymbol().adapter(ServiceVersion.class); + private static final String EVENT_FORMAT = + "{%n" + + "\"ddsource\": \"dd_debugger\",%n" + + "\"service\": \"%s\",%n" + + "\"runtimeId\": \"%s\"%n" + + "}"; + + private final String serviceName; + private final String env; + private final String version; + private final BatchUploader symbolUploader; + private final BlockingQueue scopes = new ArrayBlockingQueue<>(CAPACITY); + private final BatchUploader.MultiPartContent event; + + public SymbolSink(Config config) { + this(config, new BatchUploader(config, config.getFinalDebuggerSymDBUrl())); + } + + SymbolSink(Config config, BatchUploader symbolUploader) { + this.serviceName = TagsHelper.sanitize(config.getServiceName()); + this.env = config.getEnv(); + this.version = config.getVersion(); + this.symbolUploader = symbolUploader; + byte[] eventContent = + String.format(EVENT_FORMAT, serviceName, config.getRuntimeId()) + .getBytes(StandardCharsets.UTF_8); + this.event = new BatchUploader.MultiPartContent(eventContent, "event", "event.json"); + } + + public boolean addScope(Scope jarScope) { + ServiceVersion serviceVersion = + new ServiceVersion(serviceName, env, version, "JAVA", Collections.singletonList(jarScope)); + return scopes.offer(serviceVersion); + } + + public void flush() { + if (scopes.isEmpty()) { + return; + } + List scopesToSerialize = new ArrayList<>(); + scopes.drainTo(scopesToSerialize); + LOGGER.debug("Sending {} scopes", scopesToSerialize.size()); + for (ServiceVersion serviceVersion : scopesToSerialize) { + try { + String json = SERVICE_VERSION_ADAPTER.toJson(serviceVersion); + LOGGER.debug( + "Sending scope: {}, size={}", + serviceVersion.getScopes().get(0).getName(), + json.length()); + symbolUploader.uploadAsMultipart( + "", + event, + new BatchUploader.MultiPartContent( + json.getBytes(StandardCharsets.UTF_8), "file", "file.json")); + } catch (Exception e) { + ExceptionHelper.logException(LOGGER, e, "Error during scope serialization:"); + } + } + } +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/LanguageSpecifics.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/LanguageSpecifics.java new file mode 100644 index 00000000000..78599d993d1 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/LanguageSpecifics.java @@ -0,0 +1,144 @@ +package com.datadog.debugger.symbol; + +import com.datadog.debugger.agent.Generated; +import com.squareup.moshi.Json; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +public class LanguageSpecifics { + @Json(name = "access_modifiers") + private final List accessModifiers; + + private final List annotations; + + @Json(name = "super_class") + private final String superClass; + + private final List interfaces; + + @Json(name = "return_type") + private final String returnType; + + public LanguageSpecifics( + List accessModifiers, + List annotations, + String superClass, + List interfaces, + String returnType) { + this.accessModifiers = accessModifiers; + this.annotations = annotations; + this.superClass = superClass; + this.interfaces = interfaces; + this.returnType = returnType; + } + + public List getAccessModifiers() { + return accessModifiers; + } + + public List getAnnotations() { + return annotations; + } + + public String getSuperClass() { + return superClass; + } + + public List getInterfaces() { + return interfaces; + } + + public String getReturnType() { + return returnType; + } + + @Generated + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LanguageSpecifics that = (LanguageSpecifics) o; + return Objects.equals(accessModifiers, that.accessModifiers) + && Objects.equals(annotations, that.annotations) + && Objects.equals(superClass, that.superClass) + && Objects.equals(interfaces, that.interfaces) + && Objects.equals(returnType, that.returnType); + } + + @Generated + @Override + public int hashCode() { + return Objects.hash(accessModifiers, annotations, superClass, interfaces, returnType); + } + + @Generated + @Override + public String toString() { + return "LanguageSpecifics{" + + "accessModifiers=" + + accessModifiers + + ", annotations=" + + annotations + + ", superClass='" + + superClass + + '\'' + + ", interfaces=" + + interfaces + + ", returnType='" + + returnType + + '\'' + + '}'; + } + + public static class Builder { + private List accessModifiers; + private List annotations; + private String superClass; + private List interfaces; + private String returnType; + + public Builder addModifiers(Collection modifiers) { + if (modifiers == null || modifiers.isEmpty()) { + this.accessModifiers = null; + return this; + } + accessModifiers = new ArrayList<>(modifiers); + return this; + } + + public Builder addAnnotations(Collection annotations) { + if (annotations == null || annotations.isEmpty()) { + this.annotations = null; + return this; + } + this.annotations = new ArrayList<>(annotations); + return this; + } + + public Builder superClass(String superClass) { + this.superClass = superClass; + return this; + } + + public Builder addInterfaces(Collection interfaces) { + if (interfaces == null || interfaces.isEmpty()) { + this.interfaces = null; + return this; + } + this.interfaces = new ArrayList<>(interfaces); + return this; + } + + public Builder returnType(String returnType) { + this.returnType = returnType; + return this; + } + + public LanguageSpecifics build() { + return new LanguageSpecifics( + accessModifiers, annotations, superClass, interfaces, returnType); + } + } +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/Scope.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/Scope.java new file mode 100644 index 00000000000..bd0012bfb03 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/Scope.java @@ -0,0 +1,149 @@ +package com.datadog.debugger.symbol; + +import com.squareup.moshi.Json; +import java.util.List; + +public class Scope { + @Json(name = "scope_type") + private final ScopeType scopeType; + + @Json(name = "source_file") + private final String sourceFile; + + @Json(name = "start_line") + private final int startLine; + + @Json(name = "end_line") + private final int endLine; + + private final String name; + + @Json(name = "language_specifics") + private final LanguageSpecifics languageSpecifics; + + private final List symbols; + private final List scopes; + + public Scope( + ScopeType scopeType, + String sourceFile, + int startLine, + int endLine, + String name, + LanguageSpecifics languageSpecifics, + List symbols, + List scopes) { + this.scopeType = scopeType; + this.sourceFile = sourceFile; + this.startLine = startLine; + this.endLine = endLine; + this.name = name; + this.languageSpecifics = languageSpecifics; + this.symbols = symbols; + this.scopes = scopes; + } + + public ScopeType getScopeType() { + return scopeType; + } + + public String getSourceFile() { + return sourceFile; + } + + public int getStartLine() { + return startLine; + } + + public int getEndLine() { + return endLine; + } + + public String getName() { + return name; + } + + public LanguageSpecifics getLanguageSpecifics() { + return languageSpecifics; + } + + public List getSymbols() { + return symbols; + } + + public List getScopes() { + return scopes; + } + + @Override + public String toString() { + return "Scope{" + + "scopeType=" + + scopeType + + ", sourceFile='" + + sourceFile + + '\'' + + ", startLine=" + + startLine + + ", endLine=" + + endLine + + ", name='" + + name + + '\'' + + ", languageSpecifics=" + + languageSpecifics + + ", symbols=" + + symbols + + ", scopes=" + + scopes + + '}'; + } + + public static Builder builder( + ScopeType scopeType, String sourceFile, int startLine, int endLine) { + return new Builder(scopeType, sourceFile, startLine, endLine); + } + + public static class Builder { + private final ScopeType scopeType; + private final String sourceFile; + private final int startLine; + private final int endLine; + private String name; + private LanguageSpecifics languageSpecifics; + private List symbols; + private List scopes; + + public Builder(ScopeType scopeType, String sourceFile, int startLine, int endLine) { + this.scopeType = scopeType; + this.sourceFile = sourceFile; + this.startLine = startLine; + this.endLine = endLine; + } + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder languageSpecifics(LanguageSpecifics languageSpecifics) { + this.languageSpecifics = languageSpecifics; + return this; + } + + public Builder symbols(List symbols) { + this.symbols = symbols; + return this; + } + + public Builder scopes(List scopes) { + this.scopes = scopes; + return this; + } + + public Scope build() { + return new Scope( + scopeType, sourceFile, startLine, endLine, name, languageSpecifics, symbols, scopes); + } + } +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/ScopeType.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/ScopeType.java new file mode 100644 index 00000000000..99d8c004a55 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/ScopeType.java @@ -0,0 +1,9 @@ +package com.datadog.debugger.symbol; + +public enum ScopeType { + JAR, + CLASS, + METHOD, + LOCAL, + CLOSURE +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/ServiceVersion.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/ServiceVersion.java new file mode 100644 index 00000000000..a88fc25da63 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/ServiceVersion.java @@ -0,0 +1,25 @@ +package com.datadog.debugger.symbol; + +import java.util.List; + +public class ServiceVersion { + private final String service; + + private final String env; + private final String version; + private final String language; + private final List scopes; + + public ServiceVersion( + String service, String env, String version, String language, List scopes) { + this.service = service; + this.env = env; + this.version = version; + this.language = language; + this.scopes = scopes; + } + + public List getScopes() { + return scopes; + } +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/Symbol.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/Symbol.java new file mode 100644 index 00000000000..66deef6ee9f --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/Symbol.java @@ -0,0 +1,66 @@ +package com.datadog.debugger.symbol; + +import com.squareup.moshi.Json; + +public class Symbol { + @Json(name = "symbol_type") + private final SymbolType symbolType; + + private final String name; + private final int line; + private final String type; + + @Json(name = "language_specifics") + private final LanguageSpecifics languageSpecifics; + + public Symbol( + SymbolType symbolType, + String name, + int line, + String type, + LanguageSpecifics languageSpecifics) { + this.symbolType = symbolType; + this.name = name; + this.line = line; + this.type = type; + this.languageSpecifics = languageSpecifics; + } + + public SymbolType getSymbolType() { + return symbolType; + } + + public String getName() { + return name; + } + + public int getLine() { + return line; + } + + public String getType() { + return type; + } + + public LanguageSpecifics getLanguageSpecifics() { + return languageSpecifics; + } + + @Override + public String toString() { + return "Symbol{" + + "symbolType=" + + symbolType + + ", name='" + + name + + '\'' + + ", line=" + + line + + ", type='" + + type + + '\'' + + ", languageSpecifics=" + + languageSpecifics + + '}'; + } +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractionTransformer.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractionTransformer.java new file mode 100644 index 00000000000..203a306a1c0 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractionTransformer.java @@ -0,0 +1,140 @@ +package com.datadog.debugger.symbol; + +import com.datadog.debugger.agent.AllowListHelper; +import com.datadog.debugger.agent.Configuration; +import com.datadog.debugger.sink.SymbolSink; +import datadog.trace.api.Config; +import datadog.trace.util.AgentTaskScheduler; +import datadog.trace.util.Strings; +import java.lang.instrument.ClassFileTransformer; +import java.net.URL; +import java.security.CodeSource; +import java.security.ProtectionDomain; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SymbolExtractionTransformer implements ClassFileTransformer { + + private static final Logger LOGGER = LoggerFactory.getLogger(SymbolExtractionTransformer.class); + private static final Pattern COMMA_PATTERN = Pattern.compile(","); + + private final SymbolSink sink; + private final Map jarScopesByName = new HashMap<>(); + private final AgentTaskScheduler.Scheduled scheduled; + private final Object jarScopeLock = new Object(); + private final AllowListHelper allowListHelper; + private int totalClasses; + private final int symbolFlushThreshold; + + public SymbolExtractionTransformer() { + this(new SymbolSink(Config.get()), Config.get()); + } + + public SymbolExtractionTransformer(SymbolSink sink, Config config) { + this.sink = sink; + this.scheduled = + AgentTaskScheduler.INSTANCE.scheduleAtFixedRate( + this::flushRemainingScopes, this, 30, 30, TimeUnit.SECONDS); + String includes = config.getDebuggerSymbolIncludes(); + if (includes != null) { + this.allowListHelper = new AllowListHelper(buildFilterList(includes)); + } else { + this.allowListHelper = null; + } + this.symbolFlushThreshold = config.getDebuggerSymbolFlushThreshold(); + } + + private void flushRemainingScopes(SymbolExtractionTransformer symbolExtractionTransformer) { + synchronized (jarScopeLock) { + if (jarScopesByName.isEmpty()) { + return; + } + LOGGER.debug("Flush remaining scopes"); + addJarScope(null, true); // force flush remaining scopes + } + } + + private Configuration.FilterList buildFilterList(String includes) { + String[] includeParts = COMMA_PATTERN.split(includes); + return new Configuration.FilterList(Arrays.asList(includeParts), Collections.emptyList()); + } + + @Override + public byte[] transform( + ClassLoader loader, + String className, + Class classBeingRedefined, + ProtectionDomain protectionDomain, + byte[] classfileBuffer) { + if (className == null) { + return null; + } + if (allowListHelper == null) { + if (className.startsWith("java/") + || className.startsWith("javax/") + || className.startsWith("jdk/") + || className.startsWith("sun/") + || className.startsWith("com/sun/") + || className.startsWith("datadog/") + || className.startsWith("com/datadog/")) { + return null; + } + } else { + if (!allowListHelper.isAllowed(Strings.getClassName(className))) { + return null; + } + } + String jarName = "DEFAULT"; + if (protectionDomain != null) { + CodeSource codeSource = protectionDomain.getCodeSource(); + if (codeSource != null) { + URL location = codeSource.getLocation(); + if (location != null) { + jarName = location.getFile(); + } + } + } + LOGGER.debug("Extracting Symbols from: {}, located in: {}", className, jarName); + Scope jarScope = SymbolExtractor.extract(classfileBuffer, jarName); + addJarScope(jarScope, false); + return null; + } + + private void addJarScope(Scope jarScope, boolean forceFlush) { + List scopes = Collections.emptyList(); + synchronized (jarScopeLock) { + if (jarScope != null) { + Scope scope = jarScopesByName.get(jarScope.getName()); + if (scope != null) { + scope.getScopes().addAll(jarScope.getScopes()); + } else { + jarScopesByName.put(jarScope.getName(), jarScope); + } + totalClasses++; + } + if (totalClasses >= symbolFlushThreshold || forceFlush) { + scopes = new ArrayList<>(jarScopesByName.values()); + jarScopesByName.clear(); + totalClasses = 0; + } + } + if (!scopes.isEmpty()) { + LOGGER.debug("dumping {} jar scopes to sink", scopes.size()); + for (Scope scope : scopes) { + LOGGER.debug( + "dumping {} class scopes to sink from scope: {}", + scope.getScopes().size(), + scope.getName()); + sink.addScope(scope); + } + } + } +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractor.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractor.java new file mode 100644 index 00000000000..f5fadff6ce0 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractor.java @@ -0,0 +1,459 @@ +package com.datadog.debugger.symbol; + +import static com.datadog.debugger.instrumentation.ASMHelper.adjustLocalVarsBasedOnArgs; +import static com.datadog.debugger.instrumentation.ASMHelper.createLocalVarNodes; +import static com.datadog.debugger.instrumentation.ASMHelper.sortLocalVariables; + +import com.datadog.debugger.instrumentation.ASMHelper; +import datadog.trace.util.Strings; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.Label; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.AnnotationNode; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.FieldNode; +import org.objectweb.asm.tree.LabelNode; +import org.objectweb.asm.tree.LineNumberNode; +import org.objectweb.asm.tree.LocalVariableNode; +import org.objectweb.asm.tree.MethodNode; +import org.slf4j.LoggerFactory; + +public class SymbolExtractor { + + public static Scope extract(byte[] classFileBuffer, String jarName) { + ClassNode classNode = parseClassFile(classFileBuffer); + return extractScopes(classNode, jarName); + } + + private static Scope extractScopes(ClassNode classNode, String jarName) { + try { + String sourceFile = extractSourceFile(classNode); + List methodScopes = extractMethods(classNode, sourceFile); + int classStartLine = Integer.MAX_VALUE; + int classEndLine = 0; + for (Scope scope : methodScopes) { + classStartLine = Math.min(classStartLine, scope.getStartLine()); + classEndLine = Math.max(classEndLine, scope.getEndLine()); + } + List fields = extractFields(classNode); + LanguageSpecifics classSpecifics = + new LanguageSpecifics.Builder() + .addModifiers(extractClassModifiers(classNode.access)) + .addInterfaces(extractInterfaces(classNode)) + .addAnnotations(extractAnnotations(classNode.visibleAnnotations)) + .superClass(Strings.getClassName(classNode.superName)) + .build(); + Scope classScope = + Scope.builder(ScopeType.CLASS, sourceFile, classStartLine, classEndLine) + .name(Strings.getClassName(classNode.name)) + .scopes(methodScopes) + .symbols(fields) + .languageSpecifics(classSpecifics) + .build(); + return Scope.builder(ScopeType.JAR, jarName, 0, 0) + .name(jarName) + .scopes(new ArrayList<>(Collections.singletonList(classScope))) + .build(); + } catch (Exception ex) { + LoggerFactory.getLogger(SymbolExtractor.class).info("", ex); + return null; + } + } + + private static Collection extractInterfaces(ClassNode classNode) { + if (classNode.interfaces.isEmpty()) { + return Collections.emptyList(); + } + return classNode.interfaces.stream().map(Strings::getClassName).collect(Collectors.toList()); + } + + private static List extractFields(ClassNode classNode) { + List fields = new ArrayList<>(); + for (FieldNode fieldNode : classNode.fields) { + SymbolType symbolType = + ASMHelper.isStaticField(fieldNode) ? SymbolType.STATIC_FIELD : SymbolType.FIELD; + LanguageSpecifics fieldSpecifics = + new LanguageSpecifics.Builder() + .addModifiers(extractFieldModifiers(fieldNode.access)) + .addAnnotations(extractAnnotations(fieldNode.visibleAnnotations)) + .build(); + fields.add( + new Symbol( + symbolType, + fieldNode.name, + 0, + Type.getType(fieldNode.desc).getClassName(), + fieldSpecifics)); + } + return fields; + } + + private static List extractMethods(ClassNode classNode, String sourceFile) { + List methodScopes = new ArrayList<>(); + for (MethodNode method : classNode.methods) { + MethodLineInfo methodLineInfo = extractMethodLineInfo(method); + List varScopes = new ArrayList<>(); + List methodSymbols = new ArrayList<>(); + int localVarBaseSlot = extractArgs(method, methodSymbols, methodLineInfo.start); + extractScopesFromVariables( + sourceFile, method, methodLineInfo.lineMap, varScopes, localVarBaseSlot); + ScopeType methodScopeType = ScopeType.METHOD; + if (method.name.startsWith("lambda$")) { + methodScopeType = ScopeType.CLOSURE; + } + LanguageSpecifics methodSpecifics = + new LanguageSpecifics.Builder() + .addModifiers(extractMethodModifiers(classNode, method, method.access)) + .addAnnotations(extractAnnotations(method.visibleAnnotations)) + .returnType(Type.getType(method.desc).getReturnType().getClassName()) + .build(); + Scope methodScope = + Scope.builder(methodScopeType, sourceFile, methodLineInfo.start, methodLineInfo.end) + .name(method.name) + .scopes(varScopes) + .symbols(methodSymbols) + .languageSpecifics(methodSpecifics) + .build(); + methodScopes.add(methodScope); + } + return methodScopes; + } + + private static Collection extractClassModifiers(int access) { + List results = new ArrayList<>(); + for (int remaining = access, bit; remaining != 0; remaining -= bit) { + bit = Integer.lowestOneBit(remaining); + switch (bit) { + case Opcodes.ACC_PUBLIC: + results.add("public"); + break; + case Opcodes.ACC_PRIVATE: + results.add("private"); + break; + case Opcodes.ACC_PROTECTED: + results.add("protected"); + break; + case Opcodes.ACC_STATIC: + results.add("static"); + break; + case Opcodes.ACC_FINAL: + results.add("final"); + break; + case Opcodes.ACC_SUPER: + break; // not interesting + case Opcodes.ACC_INTERFACE: + results.add("interface"); + break; + case Opcodes.ACC_ABSTRACT: + results.add("abstract"); + break; + case Opcodes.ACC_SYNTHETIC: + results.add("synthetic"); + break; + case Opcodes.ACC_ANNOTATION: + results.add("annotation"); + break; + case Opcodes.ACC_ENUM: + results.add("enum"); + break; + default: + throw new IllegalArgumentException("Invalid access modifiers: " + bit); + } + } + return results; + } + + private static Collection extractMethodModifiers( + ClassNode classNode, MethodNode methodNode, int access) { + List results = new ArrayList<>(); + for (int remaining = access, bit; remaining != 0; remaining -= bit) { + bit = Integer.lowestOneBit(remaining); + switch (bit) { + case Opcodes.ACC_PUBLIC: + results.add("public"); + break; + case Opcodes.ACC_PRIVATE: + results.add("private"); + break; + case Opcodes.ACC_PROTECTED: + results.add("protected"); + break; + case Opcodes.ACC_STATIC: + results.add("static"); + break; + case Opcodes.ACC_FINAL: + results.add("final"); + break; + case Opcodes.ACC_SYNCHRONIZED: + results.add("synchronized"); + break; + case Opcodes.ACC_BRIDGE: + results.add("(bridge)"); + break; + case Opcodes.ACC_VARARGS: + results.add("(varargs)"); + break; + case Opcodes.ACC_NATIVE: + results.add("native"); + break; + case Opcodes.ACC_ABSTRACT: + results.add("abstract"); + break; + case Opcodes.ACC_STRICT: + results.add("strictfp"); + break; + case Opcodes.ACC_SYNTHETIC: + results.add("synthetic"); + break; + default: + throw new IllegalArgumentException("Invalid access modifiers: " + bit); + } + } + // if class is an interface && method as code this is a default method + if ((classNode.access & Opcodes.ACC_INTERFACE) > 0 && methodNode.instructions.size() > 0) { + results.add("default"); + } + return results; + } + + private static Collection extractFieldModifiers(int access) { + List results = new ArrayList<>(); + for (int remaining = access, bit; remaining != 0; remaining -= bit) { + bit = Integer.lowestOneBit(remaining); + switch (bit) { + case Opcodes.ACC_PUBLIC: + results.add("public"); + break; + case Opcodes.ACC_PRIVATE: + results.add("private"); + break; + case Opcodes.ACC_PROTECTED: + results.add("protected"); + break; + case Opcodes.ACC_STATIC: + results.add("static"); + break; + case Opcodes.ACC_FINAL: + results.add("final"); + break; + case Opcodes.ACC_VOLATILE: + results.add("volatile"); + break; + case Opcodes.ACC_TRANSIENT: + results.add("transient"); + break; + case Opcodes.ACC_SYNTHETIC: + results.add("synthetic"); + break; + case Opcodes.ACC_ENUM: + results.add("enum"); + break; + default: + throw new IllegalArgumentException("Invalid access modifiers: " + bit); + } + } + return results; + } + + private static Collection extractAnnotations(List annotationNodes) { + if (annotationNodes == null || annotationNodes.isEmpty()) { + return Collections.emptyList(); + } + List results = new ArrayList<>(); + for (AnnotationNode annotationNode : annotationNodes) { + StringBuilder sb = new StringBuilder("@"); + sb.append(Type.getType(annotationNode.desc).getClassName()); + results.add(sb.toString()); + } + return results; + } + + private static String extractSourceFile(ClassNode classNode) { + String packageName = classNode.name; + int idx = packageName.lastIndexOf('/'); + packageName = idx >= 0 ? packageName.substring(0, idx + 1) : ""; + return packageName + classNode.sourceFile; + } + + private static int extractArgs( + MethodNode method, List methodSymbols, int methodStartLine) { + boolean isStatic = (method.access & Opcodes.ACC_STATIC) != 0; + int slot = isStatic ? 0 : 1; + if (method.localVariables == null || method.localVariables.size() == 0) { + return slot; + } + Type[] argTypes = Type.getArgumentTypes(method.desc); + if (argTypes.length == 0) { + return slot; + } + List sortedLocalVars = sortLocalVariables(method.localVariables); + LocalVariableNode[] localVarsBySlot = createLocalVarNodes(sortedLocalVars); + adjustLocalVarsBasedOnArgs(isStatic, localVarsBySlot, argTypes, sortedLocalVars); + for (Type argType : argTypes) { + if (slot >= localVarsBySlot.length) { + break; + } + String argName = localVarsBySlot[slot] != null ? localVarsBySlot[slot].name : "p" + slot; + methodSymbols.add( + new Symbol(SymbolType.ARG, argName, methodStartLine, argType.getClassName(), null)); + slot += argType.getSize(); + } + return slot; + } + + private static void extractScopesFromVariables( + String sourceFile, + MethodNode methodNode, + Map monotonicLineMap, + List varScopes, + int localVarBaseSlot) { + if (methodNode.localVariables == null) { + return; + } + // using a LinkedHashMap only for having a stable order of local scopes (tests) + Map> varsByEndLabel = new LinkedHashMap<>(); + for (int i = 0; i < methodNode.localVariables.size(); i++) { + LocalVariableNode localVariable = methodNode.localVariables.get(i); + if (localVariable.index < localVarBaseSlot) { + continue; + } + varsByEndLabel.merge( + localVariable.end, + new ArrayList<>(Collections.singletonList(localVariable)), + (curr, next) -> { + curr.addAll(next); + return curr; + }); + } + List tmpScopes = new ArrayList<>(); + for (Map.Entry> entry : varsByEndLabel.entrySet()) { + List varSymbols = new ArrayList<>(); + int minLine = Integer.MAX_VALUE; + for (LocalVariableNode var : entry.getValue()) { + int line = monotonicLineMap.get(var.start.getLabel()); + minLine = Math.min(line, minLine); + varSymbols.add( + new Symbol( + SymbolType.LOCAL, var.name, line, Type.getType(var.desc).getClassName(), null)); + } + int endLine = monotonicLineMap.get(entry.getKey().getLabel()); + Scope varScope = + Scope.builder(ScopeType.LOCAL, sourceFile, minLine, endLine) + .symbols(varSymbols) + .scopes(new ArrayList<>()) + .build(); + tmpScopes.add(varScope); + } + nestScopes(varScopes, tmpScopes); + } + + private static Scope removeWidestScope(List scopes) { + Scope widestScope = null; + for (Scope scope : scopes) { + widestScope = widestScope != null ? maxScope(widestScope, scope) : scope; + } + // Remove the actual widest instance from the list, based on reference equality + scopes.remove(widestScope); + return widestScope; + } + + private static void nestScopes(List outerScopes, List scopes) { + Scope widestScope = removeWidestScope(scopes); + if (widestScope == null) { + return; + } + outerScopes.add(widestScope); + for (Scope scope : scopes) { + boolean added = false; + for (Scope outerScope : outerScopes) { + if (isInnerScope(outerScope, scope)) { + outerScope.getScopes().add(scope); + added = true; + break; + } + } + if (!added) { + outerScopes.add(scope); + } + } + for (Scope outerScope : outerScopes) { + List tmpScopes = new ArrayList<>(outerScope.getScopes()); + outerScope.getScopes().clear(); + nestScopes(outerScope.getScopes(), tmpScopes); + } + } + + private static boolean isInnerScope(Scope enclosingScope, Scope scope) { + return scope.getStartLine() >= enclosingScope.getStartLine() + && scope.getEndLine() <= enclosingScope.getEndLine(); + } + + private static Scope maxScope(Scope scope1, Scope scope2) { + return scope1.getStartLine() > scope2.getStartLine() + || scope1.getEndLine() < scope2.getEndLine() + ? scope2 + : scope1; + } + + private static int getFirstLine(MethodNode methodNode) { + AbstractInsnNode node = methodNode.instructions.getFirst(); + while (node != null) { + if (node.getType() == AbstractInsnNode.LINE) { + LineNumberNode lineNumberNode = (LineNumberNode) node; + return lineNumberNode.line; + } + node = node.getNext(); + } + return 0; + } + + private static MethodLineInfo extractMethodLineInfo(MethodNode methodNode) { + Map map = new HashMap<>(); + int startLine = getFirstLine(methodNode); + int maxLine = startLine; + AbstractInsnNode node = methodNode.instructions.getFirst(); + while (node != null) { + if (node.getType() == AbstractInsnNode.LINE) { + LineNumberNode lineNumberNode = (LineNumberNode) node; + maxLine = Math.max(lineNumberNode.line, maxLine); + } + if (node.getType() == AbstractInsnNode.LABEL) { + if (node instanceof LabelNode) { + LabelNode labelNode = (LabelNode) node; + map.put(labelNode.getLabel(), maxLine); + } + } + node = node.getNext(); + } + return new MethodLineInfo(startLine, maxLine, map); + } + + private static ClassNode parseClassFile(byte[] classfileBuffer) { + ClassReader reader = new ClassReader(classfileBuffer); + ClassNode classNode = new ClassNode(); + reader.accept(classNode, ClassReader.SKIP_FRAMES); + return classNode; + } + + public static class MethodLineInfo { + final int start; + final int end; + final Map lineMap; + + public MethodLineInfo(int start, int end, Map lineMap) { + this.start = start; + this.end = end; + this.lineMap = lineMap; + } + } +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolType.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolType.java new file mode 100644 index 00000000000..94b5b889c74 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolType.java @@ -0,0 +1,8 @@ +package com.datadog.debugger.symbol; + +public enum SymbolType { + FIELD, + STATIC_FIELD, + ARG, + LOCAL +} diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/uploader/BatchUploader.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/uploader/BatchUploader.java index 3f7a9fe813e..15dbaaf604d 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/uploader/BatchUploader.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/uploader/BatchUploader.java @@ -18,10 +18,10 @@ import java.util.concurrent.TimeoutException; import okhttp3.Call; import okhttp3.Callback; -import okhttp3.ConnectionPool; import okhttp3.Dispatcher; import okhttp3.HttpUrl; import okhttp3.MediaType; +import okhttp3.MultipartBody; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; @@ -32,6 +32,30 @@ /** Handles batching logic of upload requests sent to the intake */ public class BatchUploader { + public static class MultiPartContent { + private final byte[] content; + private final String partName; + private final String fileName; + + public MultiPartContent(byte[] content, String partName, String fileName) { + this.content = content; + this.partName = partName; + this.fileName = fileName; + } + + public byte[] getContent() { + return content; + } + + public String getPartName() { + return partName; + } + + public String getFileName() { + return fileName; + } + } + private static final Logger log = LoggerFactory.getLogger(BatchUploader.class); private static final int MINUTES_BETWEEN_ERROR_LOG = 5; private static final MediaType APPLICATION_JSON = MediaType.parse("application/json"); @@ -54,23 +78,23 @@ public class BatchUploader { private final Phaser inflightRequests = new Phaser(1); - public BatchUploader(Config config) { - this(config, new RatelimitedLogger(log, MINUTES_BETWEEN_ERROR_LOG, TimeUnit.MINUTES)); + public BatchUploader(Config config, String endpoint) { + this(config, endpoint, new RatelimitedLogger(log, MINUTES_BETWEEN_ERROR_LOG, TimeUnit.MINUTES)); } - BatchUploader(Config config, RatelimitedLogger ratelimitedLogger) { - this(config, ratelimitedLogger, ContainerInfo.get().containerId); + BatchUploader(Config config, String endpoint, RatelimitedLogger ratelimitedLogger) { + this(config, endpoint, ratelimitedLogger, ContainerInfo.get().containerId); } // Visible for testing - BatchUploader(Config config, RatelimitedLogger ratelimitedLogger, String containerId) { + BatchUploader( + Config config, String endpoint, RatelimitedLogger ratelimitedLogger, String containerId) { instrumentTheWorld = config.isDebuggerInstrumentTheWorld(); - String url = config.getFinalDebuggerSnapshotUrl(); - if (url == null || url.length() == 0) { - throw new IllegalArgumentException("Snapshot url is empty"); + if (endpoint == null || endpoint.length() == 0) { + throw new IllegalArgumentException("Endpoint url is empty"); } - urlBase = HttpUrl.get(url); - log.debug("Started SnapshotUploader with target url {}", urlBase); + urlBase = HttpUrl.get(endpoint); + log.debug("Started BatchUploader with target url {}", urlBase); apiKey = config.getApiKey(); this.ratelimitedLogger = ratelimitedLogger; responseCallback = new ResponseCallback(ratelimitedLogger, inflightRequests); @@ -84,10 +108,6 @@ public BatchUploader(Config config) { new SynchronousQueue<>(), new AgentThreadFactory(DEBUGGER_HTTP_DISPATCHER)); this.containerId = containerId; - // Reusing connections causes non daemon threads to be created which causes agent to prevent app - // from exiting. See https://github.com/square/okhttp/issues/4029 for some details. - ConnectionPool connectionPool = new ConnectionPool(MAX_RUNNING_REQUESTS, 1, TimeUnit.SECONDS); - Duration requestTimeout = Duration.ofSeconds(config.getDebuggerUploadTimeout()); client = OkHttpUtils.buildHttpClient( @@ -110,21 +130,41 @@ public void upload(byte[] batch) { } public void upload(byte[] batch, String tags) { + doUpload(() -> makeUploadRequest(batch, tags)); + } + + public void uploadAsMultipart(String tags, MultiPartContent... parts) { + doUpload(() -> makeMultipartUploadRequest(tags, parts)); + } + + private void makeMultipartUploadRequest(String tags, MultiPartContent[] parts) { + int contentLength = 0; + MultipartBody.Builder builder = new MultipartBody.Builder().setType(MultipartBody.FORM); + for (MultiPartContent part : parts) { + RequestBody fileBody = RequestBody.create(APPLICATION_JSON, part.content); + contentLength += part.content.length; + builder.addFormDataPart(part.partName, part.fileName, fileBody); + } + MultipartBody body = builder.build(); + buildAndSendRequest(body, contentLength, tags); + } + + private void doUpload(Runnable makeRequest) { if (instrumentTheWorld) { // no upload in Instrument-The-World mode return; } try { if (canEnqueueMoreRequests()) { - makeUploadRequest(batch, tags); + makeRequest.run(); debuggerMetrics.count("batch.uploaded", 1); } else { debuggerMetrics.count("request.queue.full", 1); ratelimitedLogger.warn("Cannot upload batch data: too many enqueued requests!"); } - } catch (final IllegalStateException | IOException e) { + } catch (Exception ex) { debuggerMetrics.count("batch.upload.error", 1); - ratelimitedLogger.warn("Problem uploading batch!", e); + ratelimitedLogger.warn("Problem uploading batch!", ex); } } @@ -136,11 +176,15 @@ OkHttpClient getClient() { return client; } - private void makeUploadRequest(byte[] json, String tags) throws IOException { + private void makeUploadRequest(byte[] json, String tags) { + int contentLength = json.length; // use RequestBody.create(MediaType, byte[]) to avoid changing Content-Type to // "Content-Type: application/json; charset=UTF-8" which is not recognized - int contentLength = json.length; RequestBody body = RequestBody.create(APPLICATION_JSON, json); + buildAndSendRequest(body, contentLength, tags); + } + + private void buildAndSendRequest(RequestBody body, int contentLength, String tags) { debuggerMetrics.histogram("batch.uploader.request.size", contentLength); if (log.isDebugEnabled()) { log.debug("Uploading batch data size={} bytes", contentLength); diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiHelper.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiHelper.java index 47f1828e773..31e132639a6 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiHelper.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiHelper.java @@ -46,4 +46,8 @@ public static JsonAdapter> createGenericAdapter() { ParameterizedType type = Types.newParameterizedType(Map.class, String.class, Object.class); return new Moshi.Builder().build().adapter(type); } + + public static Moshi createMoshiSymbol() { + return new Moshi.Builder().build(); + } } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiSnapshotHelper.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiSnapshotHelper.java index 7cec03f035c..5f5a9e63ee4 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiSnapshotHelper.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/MoshiSnapshotHelper.java @@ -44,7 +44,8 @@ public class MoshiSnapshotHelper { public static final String COLLECTION_SIZE_REASON = "collectionSize"; public static final String TIMEOUT_REASON = "timeout"; public static final String DEPTH_REASON = "depth"; - public static final String REDACTED_REASON = "redacted"; + public static final String REDACTED_IDENT_REASON = "redactedIdent"; + public static final String REDACTED_TYPE_REASON = "redactedType"; public static final String TYPE = "type"; public static final String VALUE = "value"; public static final String FIELDS = "fields"; @@ -462,10 +463,16 @@ public void notCaptured(SerializerWithLimits.NotCapturedReason reason) throws Ex jsonWriter.value(TIMEOUT_REASON); break; } - case REDACTED: + case REDACTED_IDENT: { jsonWriter.name(NOT_CAPTURED_REASON); - jsonWriter.value(REDACTED_REASON); + jsonWriter.value(REDACTED_IDENT_REASON); + break; + } + case REDACTED_TYPE: + { + jsonWriter.name(NOT_CAPTURED_REASON); + jsonWriter.value(REDACTED_TYPE_REASON); break; } default: diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/SerializerWithLimits.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/SerializerWithLimits.java index 68a7530a238..5d2da23f805 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/SerializerWithLimits.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/SerializerWithLimits.java @@ -50,7 +50,8 @@ enum NotCapturedReason { MAX_DEPTH, FIELD_COUNT, TIMEOUT, - REDACTED + REDACTED_IDENT, + REDACTED_TYPE } public interface TokenWriter { @@ -120,8 +121,14 @@ public void serialize(Object value, String type, Limits limits) throws Exception throw new IllegalArgumentException("Type is required for serialization"); } tokenWriter.prologue(value, type); - if (value == REDACTED_VALUE || Redaction.isRedactedType(type)) { - tokenWriter.notCaptured(NotCapturedReason.REDACTED); + NotCapturedReason reason = null; + if (value == REDACTED_VALUE) { + reason = NotCapturedReason.REDACTED_IDENT; + } else if (Redaction.isRedactedType(type)) { + reason = NotCapturedReason.REDACTED_TYPE; + } + if (reason != null) { + tokenWriter.notCaptured(reason); tokenWriter.epilogue(value); return; } diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/StringTokenWriter.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/StringTokenWriter.java index 838f1c45491..fcd89a87dc1 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/StringTokenWriter.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/StringTokenWriter.java @@ -1,5 +1,8 @@ package com.datadog.debugger.util; +import static com.datadog.debugger.util.MoshiSnapshotHelper.REDACTED_IDENT_REASON; +import static com.datadog.debugger.util.MoshiSnapshotHelper.REDACTED_TYPE_REASON; + import com.datadog.debugger.el.Value; import datadog.trace.bootstrap.debugger.EvaluationError; import java.lang.reflect.Field; @@ -155,8 +158,11 @@ public void notCaptured(SerializerWithLimits.NotCapturedReason reason) { case FIELD_COUNT: sb.append(", ..."); break; - case REDACTED: - sb.append("{redacted}"); + case REDACTED_IDENT: + sb.append("{").append(REDACTED_IDENT_REASON).append("}"); + break; + case REDACTED_TYPE: + sb.append("{").append(REDACTED_TYPE_REASON).append("}"); break; default: throw new RuntimeException("Unsupported NotCapturedReason: " + reason); diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/CapturedSnapshotTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/CapturedSnapshotTest.java index e54c151dfb1..714d89eb6f9 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/CapturedSnapshotTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/CapturedSnapshotTest.java @@ -4,7 +4,10 @@ import static com.datadog.debugger.util.MoshiSnapshotHelper.DEPTH_REASON; import static com.datadog.debugger.util.MoshiSnapshotHelper.FIELD_COUNT_REASON; import static com.datadog.debugger.util.MoshiSnapshotHelper.NOT_CAPTURED_REASON; +import static com.datadog.debugger.util.MoshiSnapshotHelper.REDACTED_IDENT_REASON; +import static com.datadog.debugger.util.MoshiSnapshotHelper.REDACTED_TYPE_REASON; import static com.datadog.debugger.util.TestHelper.setFieldInConfig; +import static datadog.trace.bootstrap.debugger.util.Redaction.REDACTED_VALUE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -35,6 +38,7 @@ import com.datadog.debugger.util.SerializerWithLimits; import com.squareup.moshi.JsonAdapter; import datadog.trace.api.Config; +import datadog.trace.api.sampling.Sampler; import datadog.trace.bootstrap.debugger.*; import datadog.trace.bootstrap.debugger.el.ValueReferences; import datadog.trace.bootstrap.debugger.util.Redaction; @@ -117,7 +121,7 @@ public void methodNotFound() throws IOException, URISyntaxException { installSingleProbe(CLASS_NAME, "foobar", null); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(2, result); + assertEquals(2, result); verify(probeStatusSink) .addError(eq(PROBE_ID), eq("Cannot find method CapturedSnapshot01::foobar")); } @@ -129,25 +133,25 @@ public void methodProbe() throws IOException, URISyntaxException { installSingleProbe(CLASS_NAME, "main", "int (java.lang.String)"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); + assertEquals(3, result); Snapshot snapshot = assertOneSnapshot(listener); - Assertions.assertNotNull(snapshot.getCaptures().getEntry()); - Assertions.assertNotNull(snapshot.getCaptures().getReturn()); + assertNotNull(snapshot.getCaptures().getEntry()); + assertNotNull(snapshot.getCaptures().getReturn()); assertCaptureArgs(snapshot.getCaptures().getEntry(), "arg", "java.lang.String", "1"); assertCaptureArgs(snapshot.getCaptures().getReturn(), "arg", "java.lang.String", "1"); assertTrue(snapshot.getDuration() > 0); assertTrue(snapshot.getStack().size() > 0); - Assertions.assertEquals("CapturedSnapshot01.main", snapshot.getStack().get(0).getFunction()); + assertEquals("CapturedSnapshot01.main", snapshot.getStack().get(0).getFunction()); } @Test public void singleLineProbe() throws IOException, URISyntaxException { final String CLASS_NAME = "CapturedSnapshot01"; DebuggerTransformerTest.TestSnapshotListener listener = - installSingleProbe(CLASS_NAME, "main", "int (java.lang.String)", "8"); + installSingleProbeAtExit(CLASS_NAME, "main", "int (java.lang.String)", "8"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); + assertEquals(3, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -155,7 +159,7 @@ public void singleLineProbe() throws IOException, URISyntaxException { assertCaptureArgs(snapshot.getCaptures().getLines().get(8), "arg", "java.lang.String", "1"); assertCaptureLocals(snapshot.getCaptures().getLines().get(8), "var1", "int", "1"); assertTrue(snapshot.getStack().size() > 0); - Assertions.assertEquals("CapturedSnapshot01.java", snapshot.getStack().get(0).getFileName()); + assertEquals("CapturedSnapshot01.java", snapshot.getStack().get(0).getFileName()); } @Test @@ -167,8 +171,8 @@ public void resolutionFails() throws IOException, URISyntaxException { DebuggerContext.init((id, clazz) -> null, null); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); - Assertions.assertEquals(0, listener.snapshots.size()); + assertEquals(3, result); + assertEquals(0, listener.snapshots.size()); } @Test @@ -186,8 +190,8 @@ public void resolutionThrows() throws IOException, URISyntaxException { null); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); - Assertions.assertEquals(0, listener.snapshots.size()); + assertEquals(3, result); + assertEquals(0, listener.snapshots.size()); } @Test @@ -197,7 +201,7 @@ public void constructor() throws IOException, URISyntaxException { installSingleProbe(CLASS_NAME, "", "(String, Object)"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "f").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); assertOneSnapshot(listener); } @@ -208,7 +212,7 @@ public void overloadedConstructor() throws IOException, URISyntaxException { installSingleProbe(CLASS_NAME, "", "()"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "f").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); assertOneSnapshot(listener); } @@ -218,7 +222,7 @@ public void veryOldClassFile() throws Exception { DebuggerTransformerTest.TestSnapshotListener listener = installSingleProbe(CLASS_NAME, "", "()"); Class testClass = Class.forName(CLASS_NAME); - Assertions.assertNotNull(testClass); + assertNotNull(testClass); testClass.newInstance(); assertOneSnapshot(listener); } @@ -229,7 +233,7 @@ public void oldJavacBug() throws Exception { DebuggerTransformerTest.TestSnapshotListener listener = installSingleProbe(CLASS_NAME, "main", null); Class testClass = Class.forName(CLASS_NAME); - Assertions.assertNotNull(testClass); + assertNotNull(testClass); int result = Reflect.on(testClass).call("main", "").get(); assertEquals(45, result); assertEquals(0, listener.snapshots.size()); @@ -242,7 +246,7 @@ public void nestedConstructor() throws Exception { installSingleProbe(CLASS_NAME, "", "(Throwable)"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "init").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); assertOneSnapshot(listener); } @@ -253,7 +257,7 @@ public void nestedConstructor2() throws Exception { installSingleProbe(CLASS_NAME, "", "(int)"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); } @@ -264,7 +268,7 @@ public void nestedConstructor3() throws Exception { installSingleProbe(CLASS_NAME, "", "(int, int, int)"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); } @@ -275,14 +279,14 @@ public void inheritedConstructor() throws Exception { installSingleProbe(CLASS_NAME + "$Inherited", "", null); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); assertCaptureFields( snapshot.getCaptures().getEntry(), "obj2", "java.lang.Object", (String) null); CapturedContext.CapturedValue obj2 = snapshot.getCaptures().getReturn().getFields().get("obj2"); Map fields = getFields(obj2); - Assertions.assertEquals(24, fields.get("intValue").getValue()); - Assertions.assertEquals(3.14, fields.get("doubleValue").getValue()); + assertEquals(24, fields.get("intValue").getValue()); + assertEquals(3.14, fields.get("doubleValue").getValue()); } @Test @@ -295,7 +299,7 @@ public void largeStackInheritedConstructor() throws Exception { createProbe(PROBE_ID2, CLASS_NAME, "", "(String, long, String)")); Class testClass = compileAndLoadClass(CLASS_NAME); long result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(4_000_000_001L, result); + assertEquals(4_000_000_001L, result); assertSnapshots(listener, 2, PROBE_ID2, PROBE_ID1); } @@ -309,7 +313,7 @@ public void multiMethods() throws IOException, URISyntaxException { createProbe(PROBE_ID2, CLASS_NAME, "f2", "(int)")); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); List snapshots = assertSnapshots(listener, 2, PROBE_ID1, PROBE_ID2); Snapshot snapshot0 = snapshots.get(0); assertCaptureArgs(snapshot0.getCaptures().getEntry(), "value", "int", "31"); @@ -328,7 +332,7 @@ public void multiProbeSameMethod() throws IOException, URISyntaxException { installProbes(CLASS_NAME, probe, probe2); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); List snapshots = assertSnapshots(listener, 2, PROBE_ID1, PROBE_ID2); Snapshot snapshot0 = snapshots.get(0); assertCaptureArgs(snapshot0.getCaptures().getEntry(), "value", "int", "31"); @@ -342,9 +346,9 @@ private List assertSnapshots( DebuggerTransformerTest.TestSnapshotListener listener, int expectedCount, ProbeId... probeIds) { - Assertions.assertEquals(expectedCount, listener.snapshots.size()); + assertEquals(expectedCount, listener.snapshots.size()); for (int i = 0; i < probeIds.length; i++) { - Assertions.assertEquals(probeIds[i].getId(), listener.snapshots.get(i).getProbe().getId()); + assertEquals(probeIds[i].getId(), listener.snapshots.get(i).getProbe().getId()); } return listener.snapshots; } @@ -356,7 +360,7 @@ public void catchBlock() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, "f", "()")); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "f").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); assertOneSnapshot(listener); } @@ -373,7 +377,7 @@ public void insideSynchronizedBlock() throws IOException, URISyntaxException { PROBE_ID, CLASS_NAME, "synchronizedBlock", "(int)", LINE_START + "-" + LINE_END)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "synchronizedBlock").get(); - Assertions.assertEquals(76, result); + assertEquals(76, result); List snapshots = assertSnapshots(listener, 10); int count = 31; for (int i = 0; i < 10; i++) { @@ -401,7 +405,7 @@ public void outsideSynchronizedBlock() throws IOException, URISyntaxException { PROBE_ID, CLASS_NAME, "synchronizedBlock", "(int)", LINE_START + "-" + LINE_END)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "synchronizedBlock").get(); - Assertions.assertEquals(76, result); + assertEquals(76, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -417,7 +421,7 @@ public void sourceFileProbe() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createSourceFileProbe(PROBE_ID, CLASS_NAME + ".java", 4)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -434,7 +438,7 @@ public void simpleSourceFileProbe() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createSourceFileProbe(PROBE_ID, "CapturedSnapshot10.java", 11)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(2, result); + assertEquals(2, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -454,7 +458,7 @@ public void sourceFileProbeFullPath() throws IOException, URISyntaxException { createSourceFileProbe(PROBE_ID, "src/main/java/" + DIR_CLASS_NAME + ".java", 11)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(2, result); + assertEquals(2, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -474,7 +478,7 @@ public void sourceFileProbeFullPathTopLevelClass() throws IOException, URISyntax createSourceFileProbe(PROBE_ID, "src/main/java/" + DIR_CLASS_NAME + ".java", 21)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(42 * 42, result); + assertEquals(42 * 42, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -496,7 +500,7 @@ public void methodProbeLineProbeMix() throws IOException, URISyntaxException { createProbe(PROBE_ID2, CLASS_NAME, "main", null)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(2, result); + assertEquals(2, result); List snapshots = assertSnapshots(listener, 2, PROBE_ID1, PROBE_ID2); Snapshot snapshot0 = snapshots.get(0); assertNull(snapshot0.getCaptures().getEntry()); @@ -504,13 +508,13 @@ public void methodProbeLineProbeMix() throws IOException, URISyntaxException { Assertions.assertEquals(1, snapshot0.getCaptures().getLines().size()); Assertions.assertEquals( "com.datadog.debugger.CapturedSnapshot11", snapshot0.getProbe().getLocation().getType()); - Assertions.assertEquals("main", snapshot0.getProbe().getLocation().getMethod()); + assertEquals("main", snapshot0.getProbe().getLocation().getMethod()); assertCaptureArgs(snapshot0.getCaptures().getLines().get(10), "arg", "java.lang.String", "2"); assertCaptureLocals(snapshot0.getCaptures().getLines().get(10), "var1", "int", "1"); Snapshot snapshot1 = snapshots.get(1); - Assertions.assertEquals( + assertEquals( "com.datadog.debugger.CapturedSnapshot11", snapshot1.getProbe().getLocation().getType()); - Assertions.assertEquals("main", snapshot1.getProbe().getLocation().getMethod()); + assertEquals("main", snapshot1.getProbe().getLocation().getMethod()); assertCaptureArgs(snapshot1.getCaptures().getEntry(), "arg", "java.lang.String", "2"); assertCaptureReturnValue(snapshot1.getCaptures().getReturn(), "int", "2"); } @@ -524,7 +528,7 @@ public void sourceFileProbeScala() throws IOException, URISyntaxException { String source = getFixtureContent("/" + FILE_NAME); Class testClass = ScalaHelper.compileAndLoad(source, CLASS_NAME, FILE_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -543,7 +547,7 @@ public void sourceFileProbeGroovy() throws IOException, URISyntaxException { GroovyClassLoader groovyClassLoader = new GroovyClassLoader(); Class testClass = groovyClassLoader.parseClass(source); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -559,13 +563,13 @@ public void sourceFileProbeKotlin() { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, createSourceFileProbe(PROBE_ID, CLASS_NAME + ".kt", 4)); URL resource = CapturedSnapshotTest.class.getResource("/" + CLASS_NAME + ".kt"); - Assertions.assertNotNull(resource); + assertNotNull(resource); List filesToDelete = new ArrayList<>(); Class testClass = KotlinHelper.compileAndLoad(CLASS_NAME, resource.getFile(), filesToDelete); try { Object companion = Reflect.on(testClass).get("Companion"); int result = Reflect.on(companion).call("main", "").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); Snapshot snapshot = assertOneSnapshot(listener); assertNull(snapshot.getCaptures().getEntry()); assertNull(snapshot.getCaptures().getReturn()); @@ -589,7 +593,7 @@ public void fieldExtractor() throws IOException, URISyntaxException { installProbes(CLASS_NAME, simpleDataProbe, compositeDataProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(143, result); + assertEquals(143, result); List snapshots = assertSnapshots(listener, 2, PROBE_ID1, PROBE_ID2); Snapshot simpleSnapshot = snapshots.get(0); Map expectedSimpleFields = new HashMap<>(); @@ -621,7 +625,7 @@ public void fieldExtractorDeep2() throws IOException, URISyntaxException { installProbes(CLASS_NAME, compositeDataProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(143, result); + assertEquals(143, result); Snapshot snapshot = assertOneSnapshot(listener); CapturedContext.CapturedValue returnValue = snapshot.getCaptures().getReturn().getLocals().get("@return"); @@ -631,11 +635,10 @@ public void fieldExtractorDeep2() throws IOException, URISyntaxException { CapturedContext.CapturedValue s1 = fields.get("s1"); Map s1Fields = (Map) s1.getValue(); - Assertions.assertEquals("101", String.valueOf(s1Fields.get("intValue").getValue())); - Assertions.assertEquals("foo1", s1Fields.get("strValue").getValue()); - Assertions.assertEquals("null", String.valueOf(s1Fields.get("listValue").getValue())); - Assertions.assertEquals( - DEPTH_REASON, String.valueOf(s1Fields.get("listValue").getNotCapturedReason())); + assertEquals("101", String.valueOf(s1Fields.get("intValue").getValue())); + assertEquals("foo1", s1Fields.get("strValue").getValue()); + assertEquals("null", String.valueOf(s1Fields.get("listValue").getValue())); + assertEquals(DEPTH_REASON, String.valueOf(s1Fields.get("listValue").getNotCapturedReason())); } @Test @@ -647,7 +650,7 @@ public void fieldExtractorLength() throws IOException, URISyntaxException { installProbes(CLASS_NAME, simpleDataProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(143, result); + assertEquals(143, result); Snapshot snapshot = assertOneSnapshot(listener); Map expectedFields = new HashMap<>(); expectedFields.put("intValue", "42"); @@ -666,14 +669,13 @@ public void fieldExtractorDisabled() throws IOException, URISyntaxException { installProbes(CLASS_NAME, simpleDataProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(143, result); + assertEquals(143, result); Snapshot snapshot = assertOneSnapshot(listener); CapturedContext.CapturedValue simpleData = snapshot.getCaptures().getReturn().getLocals().get("simpleData"); Map fields = getFields(simpleData); - Assertions.assertEquals(1, fields.size()); - Assertions.assertEquals( - DEPTH_REASON, fields.get("@" + NOT_CAPTURED_REASON).getNotCapturedReason()); + assertEquals(1, fields.size()); + assertEquals(DEPTH_REASON, fields.get("@" + NOT_CAPTURED_REASON).getNotCapturedReason()); } @Test @@ -685,13 +687,13 @@ public void fieldExtractorDepth0() throws IOException, URISyntaxException { installProbes(CLASS_NAME, simpleDataProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(143, result); + assertEquals(143, result); Snapshot snapshot = assertOneSnapshot(listener); CapturedContext.CapturedValue simpleData = snapshot.getCaptures().getReturn().getLocals().get("simpleData"); Map simpleDataFields = getFields(simpleData); - Assertions.assertEquals(1, simpleDataFields.size()); - Assertions.assertEquals( + assertEquals(1, simpleDataFields.size()); + assertEquals( DEPTH_REASON, simpleDataFields.get("@" + NOT_CAPTURED_REASON).getNotCapturedReason()); } @@ -704,15 +706,15 @@ public void fieldExtractorDepth1() throws IOException, URISyntaxException { installProbes(CLASS_NAME, simpleDataProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(143, result); + assertEquals(143, result); Snapshot snapshot = assertOneSnapshot(listener); CapturedContext.CapturedValue simpleData = snapshot.getCaptures().getReturn().getLocals().get("simpleData"); Map simpleDataFields = getFields(simpleData); - Assertions.assertEquals(4, simpleDataFields.size()); - Assertions.assertEquals("foo", simpleDataFields.get("strValue").getValue()); - Assertions.assertEquals(42, simpleDataFields.get("intValue").getValue()); - Assertions.assertEquals(DEPTH_REASON, simpleDataFields.get("listValue").getNotCapturedReason()); + assertEquals(4, simpleDataFields.size()); + assertEquals("foo", simpleDataFields.get("strValue").getValue()); + assertEquals(42, simpleDataFields.get("intValue").getValue()); + assertEquals(DEPTH_REASON, simpleDataFields.get("listValue").getNotCapturedReason()); } @Test @@ -725,29 +727,28 @@ public void fieldExtractorCount2() throws IOException, URISyntaxException { installProbes(CLASS_NAME, compositeDataProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(143, result); + assertEquals(143, result); Snapshot snapshot = assertOneSnapshot(listener); CapturedContext.CapturedValue returnValue = snapshot.getCaptures().getReturn().getLocals().get("@return"); - Assertions.assertEquals("CapturedSnapshot04$CompositeData", returnValue.getType()); + assertEquals("CapturedSnapshot04$CompositeData", returnValue.getType()); Map fields = getFields(returnValue); - Assertions.assertEquals(3, fields.size()); - Assertions.assertEquals( - FIELD_COUNT_REASON, fields.get("@" + NOT_CAPTURED_REASON).getNotCapturedReason()); + assertEquals(3, fields.size()); + assertEquals(FIELD_COUNT_REASON, fields.get("@" + NOT_CAPTURED_REASON).getNotCapturedReason()); Map s1Fields = (Map) fields.get("s1").getValue(); - Assertions.assertEquals("foo1", s1Fields.get("strValue").getValue()); - Assertions.assertEquals(101, s1Fields.get("intValue").getValue()); + assertEquals("foo1", s1Fields.get("strValue").getValue()); + assertEquals(101, s1Fields.get("intValue").getValue()); Map s2Fields = (Map) fields.get("s2").getValue(); - Assertions.assertEquals("foo2", s2Fields.get("strValue").getValue()); - Assertions.assertEquals(202, s2Fields.get("intValue").getValue()); + assertEquals("foo2", s2Fields.get("strValue").getValue()); + assertEquals(202, s2Fields.get("intValue").getValue()); CapturedContext.CapturedValue compositeData = snapshot.getCaptures().getReturn().getLocals().get("compositeData"); Map compositeDataFields = getFields(compositeData); - Assertions.assertEquals(3, compositeDataFields.size()); - Assertions.assertEquals( + assertEquals(3, compositeDataFields.size()); + assertEquals( FIELD_COUNT_REASON, compositeDataFields.get("@" + NOT_CAPTURED_REASON).getNotCapturedReason()); assertTrue(compositeDataFields.containsKey("s1")); @@ -765,7 +766,7 @@ public void uncaughtException() throws IOException, URISyntaxException { Reflect.on(testClass).call("main", "triggerUncaughtException").get(); Assertions.fail("should not reach this code"); } catch (ReflectException ex) { - Assertions.assertEquals("oops", ex.getCause().getCause().getMessage()); + assertEquals("oops", ex.getCause().getCause().getMessage()); } Snapshot snapshot = assertOneSnapshot(listener); assertCaptureThrowable( @@ -784,7 +785,7 @@ public void caughtException() throws IOException, URISyntaxException { CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, "triggerCaughtException", "()")); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "triggerCaughtException").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); assertCaptureThrowable( snapshot.getCaptures().getCaughtExceptions().get(0), @@ -808,7 +809,7 @@ public void rateLimitSnapshot() throws IOException, URISyntaxException { Class testClass = compileAndLoadClass(CLASS_NAME); for (int i = 0; i < 100; i++) { int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); + assertEquals(3, result); } assertTrue(listener.snapshots.size() < 20); } @@ -828,7 +829,7 @@ public void globalRateLimitSnapshot() throws IOException, URISyntaxException { Class testClass = compileAndLoadClass(CLASS_NAME); for (int i = 0; i < 100; i++) { int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); } assertTrue(listener.snapshots.size() < 20, "actual snapshots: " + listener.snapshots.size()); } @@ -865,11 +866,45 @@ public void simpleConditionTest() throws IOException, URISyntaxException { int result = Reflect.on(testClass).call("main", String.valueOf(i)).get(); assertTrue((i == 2 && result == 2) || result == 3); } - Assertions.assertEquals(1, listener.snapshots.size()); + assertEquals(1, listener.snapshots.size()); assertCaptureArgs( listener.snapshots.get(0).getCaptures().getReturn(), "arg", "java.lang.String", "5"); } + @Test + public void lineProbeCondition() throws IOException, URISyntaxException { + final String CLASS_NAME = "CapturedSnapshot08"; + LogProbe logProbe = + createProbeBuilder(PROBE_ID, CLASS_NAME, "doit", "int (java.lang.String)", "34") + .when( + new ProbeCondition( + DSL.when( + DSL.and( + // this is always true + DSL.and( + // this reference is resolved directly from the snapshot + DSL.eq(DSL.ref("fld"), DSL.value(11)), + // this reference chain needs to use reflection + DSL.eq( + DSL.getMember( + DSL.getMember( + DSL.getMember(DSL.ref("typed"), "fld"), "fld"), + "msg"), + DSL.value("hello"))), + DSL.eq(DSL.ref("arg"), DSL.value("5")))), + "(fld == 11 && typed.fld.fld.msg == \"hello\") && arg == '5'")) + .build(); + DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, logProbe); + Class testClass = compileAndLoadClass(CLASS_NAME); + for (int i = 0; i < 100; i++) { + int result = Reflect.on(testClass).call("main", String.valueOf(i)).get(); + assertTrue((i == 2 && result == 2) || result == 3); + } + assertEquals(1, listener.snapshots.size()); + assertCaptureArgs( + listener.snapshots.get(0).getCaptures().getLines().get(34), "arg", "java.lang.String", "5"); + } + @Test public void staticFieldCondition() throws IOException, URISyntaxException { final String CLASS_NAME = "com.datadog.debugger.CapturedSnapshot19"; @@ -883,7 +918,7 @@ public void staticFieldCondition() throws IOException, URISyntaxException { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, logProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "0").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); Map staticFields = snapshot.getCaptures().getReturn().getStaticFields(); @@ -906,8 +941,8 @@ public void simpleFalseConditionTest() throws IOException, URISyntaxException { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, logProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "0").get(); - Assertions.assertEquals(3, result); - Assertions.assertEquals(0, listener.snapshots.size()); + assertEquals(3, result); + assertEquals(0, listener.snapshots.size()); } @Test @@ -928,7 +963,7 @@ public void nullCondition() throws IOException, URISyntaxException { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, logProbes); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(1, listener.snapshots.size()); + assertEquals(1, listener.snapshots.size()); List evaluationErrors = listener.snapshots.get(0).getEvaluationErrors(); Assertions.assertEquals(1, evaluationErrors.size()); Assertions.assertEquals("nullTyped.fld.fld", evaluationErrors.get(0).getExpr()); @@ -990,8 +1025,8 @@ private List doMergedProbeConditions( installProbes(CLASS_NAME, probe1, probe2); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); - Assertions.assertEquals(expectedSnapshots, listener.snapshots.size()); + assertEquals(3, result); + assertEquals(expectedSnapshots, listener.snapshots.size()); return listener.snapshots; } @@ -1109,7 +1144,7 @@ public void fields() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, "f", "()")); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "f").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); assertCaptureFieldCount(snapshot.getCaptures().getEntry(), 5); assertCaptureFields(snapshot.getCaptures().getEntry(), "intValue", "int", "24"); @@ -1150,7 +1185,7 @@ public void inheritedFields() throws IOException, URISyntaxException { installProbes(INHERITED_CLASS_NAME, probe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "inherited").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); // Only Declared fields in the current class are captured, not inherited fields assertCaptureFieldCount(snapshot.getCaptures().getEntry(), 5); @@ -1170,7 +1205,7 @@ public void staticFields() throws IOException, URISyntaxException { installSingleProbe(CLASS_NAME, "", "()"); Class testClass = compileAndLoadClass(CLASS_NAME); long result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(4_000_000_001L, result); + assertEquals(4_000_000_001L, result); Snapshot snapshot = assertOneSnapshot(listener); Map staticFields = snapshot.getCaptures().getEntry().getStaticFields(); @@ -1195,7 +1230,7 @@ public void staticInheritedFields() throws IOException, URISyntaxException { installProbes(INHERITED_CLASS_NAME, logProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "inherited").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); Map staticFields = snapshot.getCaptures().getReturn().getStaticFields(); @@ -1215,10 +1250,10 @@ public void staticLambda() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, null, null, "33")); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "static", "email@address").get(); - Assertions.assertEquals(8, result); + assertEquals(8, result); Snapshot snapshot = assertOneSnapshot(listener); CapturedContext context = snapshot.getCaptures().getLines().get(33); - Assertions.assertNotNull(context); + assertNotNull(context); assertCaptureLocals(context, "idx", "int", "5"); } @@ -1232,10 +1267,10 @@ public void capturingLambda() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, null, null, "44")); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "capturing", "email@address").get(); - Assertions.assertEquals(8, result); + assertEquals(8, result); Snapshot snapshot = assertOneSnapshot(listener); CapturedContext context = snapshot.getCaptures().getLines().get(44); - Assertions.assertNotNull(context); + assertNotNull(context); assertCaptureLocals(context, "idx", "int", "5"); assertCaptureFields(context, "strValue", "java.lang.String", "email@address"); } @@ -1260,7 +1295,7 @@ public void tracerInstrumentedClass() throws Exception { // it's important there is no null key in this map, as Jackson is not happy about it // it's means here that argument names are not resolved correctly Assertions.assertFalse(arguments.containsKey(null)); - Assertions.assertEquals(4, arguments.size()); + assertEquals(4, arguments.size()); assertTrue(arguments.containsKey("this")); assertTrue(arguments.containsKey("apiKey")); assertTrue(arguments.containsKey("uriInfo")); @@ -1276,16 +1311,14 @@ public void noCodeMethods() throws Exception { installProbes(CLASS_NAME, nativeMethodProbe, abstractMethodProbe); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(1, result); + assertEquals(1, result); ArgumentCaptor probeIdCaptor = ArgumentCaptor.forClass(ProbeId.class); ArgumentCaptor strCaptor = ArgumentCaptor.forClass(String.class); verify(probeStatusSink, times(2)).addError(probeIdCaptor.capture(), strCaptor.capture()); - Assertions.assertEquals(PROBE_ID1.getId(), probeIdCaptor.getAllValues().get(0).getId()); - Assertions.assertEquals( - "Cannot instrument an abstract or native method", strCaptor.getAllValues().get(0)); - Assertions.assertEquals(PROBE_ID2.getId(), probeIdCaptor.getAllValues().get(1).getId()); - Assertions.assertEquals( - "Cannot instrument an abstract or native method", strCaptor.getAllValues().get(1)); + assertEquals(PROBE_ID1.getId(), probeIdCaptor.getAllValues().get(0).getId()); + assertEquals("Cannot instrument an abstract or native method", strCaptor.getAllValues().get(0)); + assertEquals(PROBE_ID2.getId(), probeIdCaptor.getAllValues().get(1).getId()); + assertEquals("Cannot instrument an abstract or native method", strCaptor.getAllValues().get(1)); } @Test @@ -1300,7 +1333,7 @@ public void duplicateClassDefinition() throws Exception { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, abstractMethodProbe); Class testClass = compileAndLoadClass(CLASS_NAME); - Assertions.assertNotNull(testClass); + assertNotNull(testClass); } @Test @@ -1310,7 +1343,7 @@ public void overloadedMethods() throws Exception { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, "overload", null)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(63, result); + assertEquals(63, result); List snapshots = assertSnapshots(listener, 4, PROBE_ID, PROBE_ID, PROBE_ID, PROBE_ID); assertCaptureReturnValue(snapshots.get(0).getCaptures().getReturn(), "int", "42"); assertCaptureArgs(snapshots.get(1).getCaptures().getEntry(), "s", "java.lang.String", "1"); @@ -1326,7 +1359,7 @@ public void noDebugInfoEmptyMethod() throws Exception { Map classFileBuffers = compile(CLASS_NAME, SourceCompiler.DebugInfo.NONE); Class testClass = loadClass(CLASS_NAME, classFileBuffers); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(48, result); + assertEquals(48, result); assertOneSnapshot(listener); } @@ -1343,8 +1376,8 @@ public void instrumentTheWorld() throws Exception { instr.removeTransformer(currentTransformer); } int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(2, result); - Assertions.assertEquals(1, listener.snapshots.size()); + assertEquals(2, result); + assertEquals(1, listener.snapshots.size()); ProbeImplementation probeImplementation = listener.snapshots.get(0).getProbe(); assertTrue(probeImplementation.isCaptureSnapshot()); assertEquals("main", probeImplementation.getLocation().getMethod()); @@ -1365,8 +1398,8 @@ public void instrumentTheWorld_excludeClass(String excludeFileName) throws Excep instr.removeTransformer(currentTransformer); } int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(2, result); - Assertions.assertEquals(0, listener.snapshots.size()); + assertEquals(2, result); + assertEquals(0, listener.snapshots.size()); } @Test @@ -1376,7 +1409,7 @@ public void objectDynamicType() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, "processWithArg", null)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(50, result); + assertEquals(50, result); Snapshot snapshot = assertOneSnapshot(listener); assertCaptureArgs(snapshot.getCaptures().getEntry(), "obj", "java.lang.Integer", "42"); assertCaptureFields( @@ -1392,7 +1425,7 @@ public void exceptionAsLocalVariable() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, null, null, "14")); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(42, result); + assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); Map expectedFields = new HashMap<>(); expectedFields.put("detailMessage", "For input string: \"a\""); @@ -1415,7 +1448,7 @@ public void evaluateAtEntry() throws IOException, URISyntaxException { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, logProbes); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); + assertEquals(3, result); assertOneSnapshot(listener); } @@ -1432,7 +1465,7 @@ public void evaluateAtExit() throws IOException, URISyntaxException { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, logProbes); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); + assertEquals(3, result); assertOneSnapshot(listener); } @@ -1449,10 +1482,10 @@ public void evaluateAtExitFalse() throws IOException, URISyntaxException { DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, logProbes); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); - Assertions.assertEquals(0, listener.snapshots.size()); + assertEquals(3, result); + assertEquals(0, listener.snapshots.size()); assertTrue(listener.skipped); - Assertions.assertEquals(DebuggerContext.SkipCause.CONDITION, listener.cause); + assertEquals(DebuggerContext.SkipCause.CONDITION, listener.cause); } @Test @@ -1469,7 +1502,7 @@ public void uncaughtExceptionConditionLocalVar() throws IOException, URISyntaxEx Reflect.on(testClass).call("main", "triggerUncaughtException").get(); Assertions.fail("should not reach this code"); } catch (ReflectException ex) { - Assertions.assertEquals("oops", ex.getCause().getCause().getMessage()); + assertEquals("oops", ex.getCause().getCause().getMessage()); } Snapshot snapshot = assertOneSnapshot(listener); assertCaptureThrowable( @@ -1478,10 +1511,9 @@ public void uncaughtExceptionConditionLocalVar() throws IOException, URISyntaxEx "oops", "CapturedSnapshot05.triggerUncaughtException", 7); - Assertions.assertEquals(2, snapshot.getEvaluationErrors().size()); - Assertions.assertEquals( - "Cannot find symbol: after", snapshot.getEvaluationErrors().get(0).getMessage()); - Assertions.assertEquals( + assertEquals(2, snapshot.getEvaluationErrors().size()); + assertEquals("Cannot find symbol: after", snapshot.getEvaluationErrors().get(0).getMessage()); + assertEquals( "java.lang.IllegalStateException: oops", snapshot.getEvaluationErrors().get(1).getMessage()); } @@ -1494,7 +1526,7 @@ public void enumConstructorArgs() throws IOException, URISyntaxException { installProbes(ENUM_CLASS, createProbe(PROBE_ID, ENUM_CLASS, "", null)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(2, result); + assertEquals(2, result); assertSnapshots(listener, 3, PROBE_ID); Map arguments = listener.snapshots.get(0).getCaptures().getEntry().getArguments(); @@ -1511,7 +1543,7 @@ public void enumValues() throws IOException, URISyntaxException { installProbes(CLASS_NAME, createProbe(PROBE_ID, CLASS_NAME, "convert", null)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "2").get(); - Assertions.assertEquals(2, result); + assertEquals(2, result); Snapshot snapshot = assertOneSnapshot(listener); assertCaptureReturnValue( snapshot.getCaptures().getReturn(), @@ -1527,7 +1559,7 @@ public void recursiveCapture() throws IOException, URISyntaxException { installProbes(INNER_CLASS, createProbe(PROBE_ID, INNER_CLASS, "size", null)); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "").get(); - Assertions.assertEquals(1, result); + assertEquals(1, result); } @Test @@ -1541,7 +1573,7 @@ public void recursiveCaptureException() throws IOException, URISyntaxException { Reflect.on(testClass).call("main", "exception").get(); Assertions.fail("should not reach this code"); } catch (ReflectException ex) { - Assertions.assertEquals("not supported", ex.getCause().getCause().getMessage()); + assertEquals("not supported", ex.getCause().getCause().getMessage()); } } @@ -1584,10 +1616,10 @@ private Snapshot doUnknownCount(String CLASS_NAME) throws IOException, URISyntax public void beforeForLoopLineProbe() throws IOException, URISyntaxException { final String CLASS_NAME = "CapturedSnapshot02"; DebuggerTransformerTest.TestSnapshotListener listener = - installSingleProbe(CLASS_NAME, null, null, "46"); + installSingleProbeAtExit(CLASS_NAME, null, null, "46"); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "synchronizedBlock").get(); - Assertions.assertEquals(76, result); + assertEquals(76, result); Snapshot snapshot = assertOneSnapshot(listener); assertCaptureLocals(snapshot.getCaptures().getLines().get(46), "count", "int", "31"); } @@ -1599,16 +1631,18 @@ public void dupLineProbeSameTemplate() throws IOException, URISyntaxException { LogProbe probe1 = createProbeBuilder(PROBE_ID1, CLASS_NAME, null, null, "39") .template(LOG_TEMPLATE, parseTemplate(LOG_TEMPLATE)) + .evaluateAt(MethodLocation.EXIT) .build(); LogProbe probe2 = createProbeBuilder(PROBE_ID2, CLASS_NAME, null, null, "39") .template(LOG_TEMPLATE, parseTemplate(LOG_TEMPLATE)) + .evaluateAt(MethodLocation.EXIT) .build(); DebuggerTransformerTest.TestSnapshotListener listener = installProbes(CLASS_NAME, probe1, probe2); Class testClass = compileAndLoadClass(CLASS_NAME); int result = Reflect.on(testClass).call("main", "1").get(); - Assertions.assertEquals(3, result); + assertEquals(3, result); List snapshots = assertSnapshots(listener, 2, PROBE_ID1, PROBE_ID2); for (Snapshot snapshot : snapshots) { assertEquals("msg1=hello", snapshot.getMessage()); @@ -1632,17 +1666,23 @@ public void keywordRedaction() throws IOException, URISyntaxException { Assertions.assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); assertEquals( - "arg=secret123 secret={Could not evaluate the expression because 'secret' was redacted} password={Could not evaluate the expression because 'this.password' was redacted} fromMap={Could not evaluate the expression because 'strMap[\"password\"]' was redacted}", + "arg=secret123 secret={" + + REDACTED_VALUE + + "} password={" + + REDACTED_VALUE + + "} fromMap={" + + REDACTED_VALUE + + "}", snapshot.getMessage()); CapturedContext.CapturedValue secretLocalVar = snapshot.getCaptures().getReturn().getLocals().get("secret"); CapturedContext.CapturedValue secretValued = VALUE_ADAPTER.fromJson(secretLocalVar.getStrValue()); - assertEquals("redacted", secretValued.getNotCapturedReason()); + assertEquals(REDACTED_IDENT_REASON, secretValued.getNotCapturedReason()); Map thisFields = getFields(snapshot.getCaptures().getReturn().getArguments().get("this")); CapturedContext.CapturedValue passwordField = thisFields.get("password"); - assertEquals("redacted", passwordField.getNotCapturedReason()); + assertEquals(REDACTED_IDENT_REASON, passwordField.getNotCapturedReason()); Map strMap = (Map) thisFields.get("strMap").getValue(); assertNull(strMap.get("password")); } @@ -1742,12 +1782,20 @@ public void typeRedactionSnapshot() throws IOException, URISyntaxException { Assertions.assertEquals(42, result); Snapshot snapshot = assertOneSnapshot(listener); assertEquals( - "arg=secret123 credentials={Could not evaluate the expression because 'creds' was redacted} user={Could not evaluate the expression because 'this.creds' was redacted} code={Could not evaluate the expression because 'creds' was redacted} dave={Could not evaluate the expression because 'credMap[\"dave\"]' was redacted}", + "arg=secret123 credentials={" + + REDACTED_VALUE + + "} user={" + + REDACTED_VALUE + + "} code={" + + REDACTED_VALUE + + "} dave={" + + REDACTED_VALUE + + "}", snapshot.getMessage()); Map thisFields = getFields(snapshot.getCaptures().getReturn().getArguments().get("this")); CapturedContext.CapturedValue credsField = thisFields.get("creds"); - assertEquals("redacted", credsField.getNotCapturedReason()); + assertEquals(REDACTED_TYPE_REASON, credsField.getNotCapturedReason()); Map credMap = (Map) thisFields.get("credMap").getValue(); assertNull(credMap.get("dave")); } @@ -1811,6 +1859,51 @@ public void typeRedactionCondition() throws IOException, URISyntaxException { snapshots.get(2).getEvaluationErrors().get(0).getMessage()); } + @Test + public void ensureCallingSamplingMethodProbe() throws IOException, URISyntaxException { + doSamplingTest(this::methodProbe, 1, 1); + } + + @Test + public void ensureCallingSamplingProbeCondition() throws IOException, URISyntaxException { + doSamplingTest(this::simpleConditionTest, 1, 1); + } + + @Test + public void ensureCallingSamplingDupMethodProbeCondition() + throws IOException, URISyntaxException { + doSamplingTest(this::mergedProbesWithAdditionalProbeConditionTest, 2, 2); + } + + @Test + public void ensureCallingSamplingLineProbe() throws IOException, URISyntaxException { + doSamplingTest(this::singleLineProbe, 1, 1); + } + + @Test + public void ensureCallingSamplingLineProbeCondition() throws IOException, URISyntaxException { + doSamplingTest(this::lineProbeCondition, 1, 1); + } + + interface TestMethod { + void run() throws IOException, URISyntaxException; + } + + private void doSamplingTest(TestMethod testRun, int expectedGlobalCount, int expectedProbeCount) + throws IOException, URISyntaxException { + MockSampler probeSampler = new MockSampler(); + MockSampler globalSampler = new MockSampler(); + ProbeRateLimiter.setSamplerSupplier(rate -> rate < 101 ? probeSampler : globalSampler); + ProbeRateLimiter.setGlobalSnapshotRate(1000); + try { + testRun.run(); + } finally { + ProbeRateLimiter.setSamplerSupplier(null); + } + assertEquals(expectedGlobalCount, globalSampler.callCount); + assertEquals(expectedProbeCount, probeSampler.callCount); + } + private DebuggerTransformerTest.TestSnapshotListener setupInstrumentTheWorldTransformer( String excludeFileName) { Config config = mock(Config.class); @@ -1820,6 +1913,7 @@ private DebuggerTransformerTest.TestSnapshotListener setupInstrumentTheWorldTran when(config.getDebuggerExcludeFiles()).thenReturn(excludeFileName); when(config.getFinalDebuggerSnapshotUrl()) .thenReturn("http://localhost:8126/debugger/v1/input"); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); when(config.getDebuggerUploadBatchSize()).thenReturn(100); DebuggerTransformerTest.TestSnapshotListener listener = new DebuggerTransformerTest.TestSnapshotListener(); @@ -1846,9 +1940,9 @@ private void setCorrelationSingleton(Object instance) { } private Snapshot assertOneSnapshot(DebuggerTransformerTest.TestSnapshotListener listener) { - Assertions.assertEquals(1, listener.snapshots.size()); + assertEquals(1, listener.snapshots.size()); Snapshot snapshot = listener.snapshots.get(0); - Assertions.assertEquals(PROBE_ID.getId(), snapshot.getProbe().getId()); + assertEquals(PROBE_ID.getId(), snapshot.getProbe().getId()); return snapshot; } @@ -1858,6 +1952,12 @@ private DebuggerTransformerTest.TestSnapshotListener installSingleProbe( return installProbes(typeName, logProbes); } + private DebuggerTransformerTest.TestSnapshotListener installSingleProbeAtExit( + String typeName, String methodName, String signature, String... lines) { + LogProbe logProbes = createProbeAtExit(PROBE_ID, typeName, methodName, signature, lines); + return installProbes(typeName, logProbes); + } + private DebuggerTransformerTest.TestSnapshotListener installProbes( String expectedClassName, Configuration configuration) { Config config = mock(Config.class); @@ -1866,6 +1966,7 @@ private DebuggerTransformerTest.TestSnapshotListener installProbes( when(config.isDebuggerVerifyByteCode()).thenReturn(true); when(config.getFinalDebuggerSnapshotUrl()) .thenReturn("http://localhost:8126/debugger/v1/input"); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); Collection logProbes = configuration.getLogProbes(); instrumentationListener = new MockInstrumentationListener(); probeStatusSink = mock(ProbeStatusSink.class); @@ -1897,7 +1998,7 @@ private DebuggerTransformerTest.TestSnapshotListener installProbes( private ProbeImplementation resolver( String id, Class callingClass, String expectedClassName, Collection logProbes) { - Assertions.assertEquals(expectedClassName, callingClass.getName()); + assertEquals(expectedClassName, callingClass.getName()); for (LogProbe probe : logProbes) { if (probe.getId().equals(id)) { return probe; @@ -1919,30 +2020,29 @@ private DebuggerTransformerTest.TestSnapshotListener installProbes( private void assertCaptureArgs( CapturedContext context, String name, String typeName, String value) { CapturedContext.CapturedValue capturedValue = context.getArguments().get(name); - Assertions.assertEquals(typeName, capturedValue.getType()); - Assertions.assertEquals(value, getValue(capturedValue)); + assertEquals(typeName, capturedValue.getType()); + assertEquals(value, getValue(capturedValue)); } private void assertCaptureLocals( CapturedContext context, String name, String typeName, String value) { CapturedContext.CapturedValue localVar = context.getLocals().get(name); - Assertions.assertEquals(typeName, localVar.getType()); - Assertions.assertEquals(value, getValue(localVar)); + assertEquals(typeName, localVar.getType()); + assertEquals(value, getValue(localVar)); } private void assertCaptureLocals( CapturedContext context, String name, String typeName, Map expectedFields) { CapturedContext.CapturedValue localVar = context.getLocals().get(name); - Assertions.assertEquals(typeName, localVar.getType()); + assertEquals(typeName, localVar.getType()); Map fields = getFields(localVar); for (Map.Entry entry : expectedFields.entrySet()) { assertTrue(fields.containsKey(entry.getKey())); CapturedContext.CapturedValue fieldCapturedValue = fields.get(entry.getKey()); if (fieldCapturedValue.getNotCapturedReason() != null) { - Assertions.assertEquals( - entry.getValue(), String.valueOf(fieldCapturedValue.getNotCapturedReason())); + assertEquals(entry.getValue(), String.valueOf(fieldCapturedValue.getNotCapturedReason())); } else { - Assertions.assertEquals(entry.getValue(), String.valueOf(fieldCapturedValue.getValue())); + assertEquals(entry.getValue(), String.valueOf(fieldCapturedValue.getValue())); } } } @@ -1950,18 +2050,18 @@ private void assertCaptureLocals( private void assertCaptureFields( CapturedContext context, String name, String typeName, String value) { CapturedContext.CapturedValue field = context.getFields().get(name); - Assertions.assertEquals(typeName, field.getType()); - Assertions.assertEquals(value, getValue(field)); + assertEquals(typeName, field.getType()); + assertEquals(value, getValue(field)); } private void assertCaptureFields( CapturedContext context, String name, String typeName, Collection collection) { CapturedContext.CapturedValue field = context.getFields().get(name); - Assertions.assertEquals(typeName, field.getType()); + assertEquals(typeName, field.getType()); Iterator iterator = collection.iterator(); for (Object obj : getCollection(field)) { if (iterator.hasNext()) { - Assertions.assertEquals(iterator.next(), obj); + assertEquals(iterator.next(), obj); } else { Assertions.fail("not same number of elements"); } @@ -1971,38 +2071,37 @@ private void assertCaptureFields( private void assertCaptureFields( CapturedContext context, String name, String typeName, Map expectedMap) { CapturedContext.CapturedValue field = context.getFields().get(name); - Assertions.assertEquals(typeName, field.getType()); + assertEquals(typeName, field.getType()); Map map = getMap(field); - Assertions.assertEquals(expectedMap.size(), map.size()); + assertEquals(expectedMap.size(), map.size()); for (Map.Entry entry : map.entrySet()) { assertTrue(expectedMap.containsKey(entry.getKey())); - Assertions.assertEquals(expectedMap.get(entry.getKey()), entry.getValue()); + assertEquals(expectedMap.get(entry.getKey()), entry.getValue()); } } private void assertCaptureFieldCount(CapturedContext context, int expectedFieldCount) { - Assertions.assertEquals(expectedFieldCount, context.getFields().size()); + assertEquals(expectedFieldCount, context.getFields().size()); } private void assertCaptureReturnValue(CapturedContext context, String typeName, String value) { CapturedContext.CapturedValue returnValue = context.getLocals().get("@return"); - Assertions.assertEquals(typeName, returnValue.getType()); - Assertions.assertEquals(value, getValue(returnValue)); + assertEquals(typeName, returnValue.getType()); + assertEquals(value, getValue(returnValue)); } private void assertCaptureReturnValue( CapturedContext context, String typeName, Map expectedFields) { CapturedContext.CapturedValue returnValue = context.getLocals().get("@return"); - Assertions.assertEquals(typeName, returnValue.getType()); + assertEquals(typeName, returnValue.getType()); Map fields = getFields(returnValue); for (Map.Entry entry : expectedFields.entrySet()) { assertTrue(fields.containsKey(entry.getKey())); CapturedContext.CapturedValue fieldCapturedValue = fields.get(entry.getKey()); if (fieldCapturedValue.getNotCapturedReason() != null) { - Assertions.assertEquals( - entry.getValue(), String.valueOf(fieldCapturedValue.getNotCapturedReason())); + assertEquals(entry.getValue(), String.valueOf(fieldCapturedValue.getNotCapturedReason())); } else { - Assertions.assertEquals(entry.getValue(), String.valueOf(fieldCapturedValue.getValue())); + assertEquals(entry.getValue(), String.valueOf(fieldCapturedValue.getValue())); } } } @@ -2019,13 +2118,13 @@ private void assertCaptureThrowable( String message, String methodName, int lineNumber) { - Assertions.assertNotNull(throwable); - Assertions.assertEquals(typeName, throwable.getType()); - Assertions.assertEquals(message, throwable.getMessage()); - Assertions.assertNotNull(throwable.getStacktrace()); + assertNotNull(throwable); + assertEquals(typeName, throwable.getType()); + assertEquals(message, throwable.getMessage()); + assertNotNull(throwable.getStacktrace()); Assertions.assertFalse(throwable.getStacktrace().isEmpty()); - Assertions.assertEquals(methodName, throwable.getStacktrace().get(0).getFunction()); - Assertions.assertEquals(lineNumber, throwable.getStacktrace().get(0).getLineNumber()); + assertEquals(methodName, throwable.getStacktrace().get(0).getFunction()); + assertEquals(lineNumber, throwable.getStacktrace().get(0).getLineNumber()); } private static String getValue(CapturedContext.CapturedValue capturedValue) { @@ -2118,6 +2217,13 @@ private static LogProbe createProbe( return createProbeBuilder(id, typeName, methodName, signature, lines).build(); } + private static LogProbe createProbeAtExit( + ProbeId id, String typeName, String methodName, String signature, String... lines) { + return createProbeBuilder(id, typeName, methodName, signature, lines) + .evaluateAt(MethodLocation.EXIT) + .build(); + } + private static LogProbe.Builder createProbeBuilder( ProbeId id, String typeName, String methodName, String signature, String... lines) { return LogProbe.builder() @@ -2135,6 +2241,7 @@ private static LogProbe createSourceFileProbe(ProbeId id, String sourceFile, int .probeId(id) .captureSnapshot(true) .where(null, null, null, line, sourceFile) + .evaluateAt(MethodLocation.EXIT) .build(); } @@ -2147,6 +2254,27 @@ public void instrumentationResult(ProbeDefinition definition, InstrumentationRes } } + static class MockSampler implements Sampler { + + int callCount; + + @Override + public boolean sample() { + callCount++; + return true; + } + + @Override + public boolean keep() { + return false; + } + + @Override + public boolean drop() { + return false; + } + } + static class KotlinHelper { public static Class compileAndLoad( String className, String sourceFileName, List outputFilesToDelete) { diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/ConfigurationUpdaterTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/ConfigurationUpdaterTest.java index 30749fdf549..5261dfb892f 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/ConfigurationUpdaterTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/ConfigurationUpdaterTest.java @@ -60,6 +60,7 @@ public class ConfigurationUpdaterTest { void setUp() { lenient().when(tracerConfig.getFinalDebuggerSnapshotUrl()).thenReturn("http://localhost"); lenient().when(tracerConfig.getDebuggerUploadBatchSize()).thenReturn(100); + lenient().when(tracerConfig.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost"); debuggerSinkWithMockStatusSink = new DebuggerSink(tracerConfig, probeStatusSink); } diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/DebuggerTransformerTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/DebuggerTransformerTest.java index e4be8e3bb89..a10ba941959 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/DebuggerTransformerTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/DebuggerTransformerTest.java @@ -353,6 +353,7 @@ private Config createConfig() { Config config = mock(Config.class); when(config.getFinalDebuggerSnapshotUrl()) .thenReturn("http://localhost:8126/debugger/v1/input"); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); when(config.getDebuggerUploadBatchSize()).thenReturn(100); return config; } diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/LogProbesInstrumentationTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/LogProbesInstrumentationTest.java index 8b5172d6579..47f80e0c94d 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/LogProbesInstrumentationTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/LogProbesInstrumentationTest.java @@ -488,6 +488,7 @@ private DebuggerTransformerTest.TestSnapshotListener installProbes( when(config.isDebuggerClassFileDumpEnabled()).thenReturn(true); when(config.getFinalDebuggerSnapshotUrl()) .thenReturn("http://localhost:8126/debugger/v1/input"); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); when(config.getDebuggerUploadBatchSize()).thenReturn(100); currentTransformer = new DebuggerTransformer(config, configuration); instr.addTransformer(currentTransformer); diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/MetricProbesInstrumentationTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/MetricProbesInstrumentationTest.java index 8a29495bfe1..d898d7d904a 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/MetricProbesInstrumentationTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/MetricProbesInstrumentationTest.java @@ -1179,6 +1179,7 @@ private MetricForwarderListener installMetricProbes(Configuration configuration) when(config.isDebuggerClassFileDumpEnabled()).thenReturn(true); when(config.getFinalDebuggerSnapshotUrl()) .thenReturn("http://localhost:8126/debugger/v1/input"); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); probeStatusSink = mock(ProbeStatusSink.class); currentTransformer = new DebuggerTransformer( diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanDecorationProbeInstrumentationTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanDecorationProbeInstrumentationTest.java index 2eb09c842c0..b19401ac7a2 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanDecorationProbeInstrumentationTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanDecorationProbeInstrumentationTest.java @@ -584,6 +584,7 @@ private void installSpanDecorationProbes(String expectedClassName, Configuration when(config.isDebuggerClassFileDumpEnabled()).thenReturn(true); when(config.getFinalDebuggerSnapshotUrl()) .thenReturn("http://localhost:8126/debugger/v1/input"); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); probeStatusSink = mock(ProbeStatusSink.class); currentTransformer = new DebuggerTransformer( diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanProbeInstrumentationTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanProbeInstrumentationTest.java index 4823ff4dec1..0764e1a5734 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanProbeInstrumentationTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/SpanProbeInstrumentationTest.java @@ -145,6 +145,7 @@ private MockTracer installSpanProbes(Configuration configuration) { when(config.isDebuggerVerifyByteCode()).thenReturn(true); when(config.getFinalDebuggerSnapshotUrl()) .thenReturn("http://localhost:8126/debugger/v1/input"); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); probeStatusSink = mock(ProbeStatusSink.class); currentTransformer = new DebuggerTransformer( diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/DebuggerSinkTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/DebuggerSinkTest.java index a0df890b0c9..61a40fa136f 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/DebuggerSinkTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/DebuggerSinkTest.java @@ -71,6 +71,7 @@ void setUp() { when(config.getEnv()).thenReturn("test"); when(config.getVersion()).thenReturn("foo"); when(config.getDebuggerUploadBatchSize()).thenReturn(1); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); EXPECTED_SNAPSHOT_TAGS = "^env:test,version:foo,debugger_version:\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?~[0-9a-f]+,agent_version:null,host_name:" diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/SymbolSinkTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/SymbolSinkTest.java new file mode 100644 index 00000000000..d158780d92e --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/SymbolSinkTest.java @@ -0,0 +1,53 @@ +package com.datadog.debugger.sink; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.datadog.debugger.symbol.Scope; +import com.datadog.debugger.symbol.ScopeType; +import com.datadog.debugger.uploader.BatchUploader; +import datadog.trace.api.Config; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +class SymbolSinkTest { + + @Test + public void testFlush() { + SymbolUploaderMock symbolUploaderMock = new SymbolUploaderMock(); + Config config = mock(Config.class); + when(config.getServiceName()).thenReturn("service1"); + SymbolSink symbolSink = new SymbolSink(config, symbolUploaderMock); + symbolSink.addScope(Scope.builder(ScopeType.JAR, null, 0, 0).build()); + symbolSink.flush(); + assertEquals(2, symbolUploaderMock.multiPartContents.size()); + BatchUploader.MultiPartContent eventContent = symbolUploaderMock.multiPartContents.get(0); + assertEquals("event", eventContent.getPartName()); + assertEquals("event.json", eventContent.getFileName()); + String strEventContent = new String(eventContent.getContent()); + assertTrue(strEventContent.contains("\"ddsource\": \"dd_debugger\"")); + assertTrue(strEventContent.contains("\"service\": \"service1\"")); + BatchUploader.MultiPartContent symbolContent = symbolUploaderMock.multiPartContents.get(1); + assertEquals("file", symbolContent.getPartName()); + assertEquals("file.json", symbolContent.getFileName()); + assertEquals( + "{\"language\":\"JAVA\",\"scopes\":[{\"end_line\":0,\"scope_type\":\"JAR\",\"start_line\":0}],\"service\":\"service1\"}", + new String(symbolContent.getContent())); + } + + static class SymbolUploaderMock extends BatchUploader { + final List multiPartContents = new ArrayList<>(); + + public SymbolUploaderMock() { + super(Config.get(), "http://localhost"); + } + + @Override + public void uploadAsMultipart(String tags, MultiPartContent... parts) { + multiPartContents.addAll(Arrays.asList(parts)); + } + } +} diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java new file mode 100644 index 00000000000..b3fda99615f --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java @@ -0,0 +1,898 @@ +package com.datadog.debugger.symbol; + +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.when; +import static utils.InstrumentationTestHelper.compileAndLoadClass; + +import com.datadog.debugger.sink.SymbolSink; +import datadog.trace.api.Config; +import java.io.IOException; +import java.lang.instrument.Instrumentation; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; +import net.bytebuddy.agent.ByteBuddyAgent; +import org.joor.Reflect; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class SymbolExtractionTransformerTest { + private static final String SYMBOL_PACKAGE = "com.datadog.debugger.symbol."; + private static final String SYMBOL_PACKAGE_DIR = SYMBOL_PACKAGE.replace('.', '/'); + + private Instrumentation instr = ByteBuddyAgent.install(); + private Config config; + + @BeforeEach + public void setUp() { + config = Mockito.mock(Config.class); + when(config.getDebuggerSymbolIncludes()).thenReturn(SYMBOL_PACKAGE); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); + when(config.getDebuggerSymbolFlushThreshold()).thenReturn(1); + } + + @Test + public void noIncludesFilterOutDatadogClass() throws IOException, URISyntaxException { + config = Mockito.mock(Config.class); + when(config.getFinalDebuggerSymDBUrl()).thenReturn("http://localhost:8126/symdb/v1/input"); + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction01"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + assertFalse( + symbolSinkMock.jarScopes.stream() + .flatMap(scope -> scope.getScopes().stream()) + .anyMatch(scope -> scope.getName().equals(CLASS_NAME))); + } + + @Test + public void symbolExtraction01() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction01"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction01.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 2, 20, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 2, 2, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 4, 20, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 4); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 4, 20, SOURCE_FILE, 1, 2); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 4); + assertSymbol( + mainMethodLocalScope.getSymbols().get(1), + SymbolType.LOCAL, + "var3", + Integer.TYPE.getTypeName(), + 19); + Scope ifLine5Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(ifLine5Scope, ScopeType.LOCAL, null, 6, 17, SOURCE_FILE, 1, 1); + assertSymbol( + ifLine5Scope.getSymbols().get(0), SymbolType.LOCAL, "var2", Integer.TYPE.getTypeName(), 6); + Scope forLine7Scope = ifLine5Scope.getScopes().get(0); + assertScope(forLine7Scope, ScopeType.LOCAL, null, 7, 15, SOURCE_FILE, 1, 1); + assertSymbol( + forLine7Scope.getSymbols().get(0), SymbolType.LOCAL, "i", Integer.TYPE.getTypeName(), 7); + Scope forBodyLine7Scope = forLine7Scope.getScopes().get(0); + assertScope(forBodyLine7Scope, ScopeType.LOCAL, null, 8, 15, SOURCE_FILE, 1, 3); + assertSymbol( + forBodyLine7Scope.getSymbols().get(0), + SymbolType.LOCAL, + "foo", + Integer.TYPE.getTypeName(), + 8); + assertSymbol( + forBodyLine7Scope.getSymbols().get(1), + SymbolType.LOCAL, + "bar", + Integer.TYPE.getTypeName(), + 9); + assertSymbol( + forBodyLine7Scope.getSymbols().get(2), + SymbolType.LOCAL, + "j", + Integer.TYPE.getTypeName(), + 11); + Scope whileLine12 = forBodyLine7Scope.getScopes().get(0); + assertScope(whileLine12, ScopeType.LOCAL, null, 13, 14, SOURCE_FILE, 0, 1); + assertSymbol( + whileLine12.getSymbols().get(0), SymbolType.LOCAL, "var4", Integer.TYPE.getTypeName(), 13); + } + + @Test + public void symbolExtraction02() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction02"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction02.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 6, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 3, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 5, 6, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 5); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 5, 6, SOURCE_FILE, 0, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + String.class.getTypeName(), + 5); + } + + @Test + public void symbolExtraction03() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction03"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction03.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 4, 28, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 4, 4, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 6, 28, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 6); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 6, 28, SOURCE_FILE, 2, 2); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + String.class.getTypeName(), + 6); + assertSymbol( + mainMethodLocalScope.getSymbols().get(1), + SymbolType.LOCAL, + "var5", + String.class.getTypeName(), + 27); + Scope elseLine10Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(elseLine10Scope, ScopeType.LOCAL, null, 12, 24, SOURCE_FILE, 1, 4); + assertSymbol( + elseLine10Scope.getSymbols().get(0), + SymbolType.LOCAL, + "var31", + String.class.getTypeName(), + 12); + assertSymbol( + elseLine10Scope.getSymbols().get(1), + SymbolType.LOCAL, + "var32", + String.class.getTypeName(), + 13); + assertSymbol( + elseLine10Scope.getSymbols().get(2), + SymbolType.LOCAL, + "var30", + String.class.getTypeName(), + 15); + assertSymbol( + elseLine10Scope.getSymbols().get(3), + SymbolType.LOCAL, + "var3", + String.class.getTypeName(), + 17); + Scope ifLine19Scope = elseLine10Scope.getScopes().get(0); + assertScope(ifLine19Scope, ScopeType.LOCAL, null, 20, 21, SOURCE_FILE, 0, 1); + assertSymbol( + ifLine19Scope.getSymbols().get(0), + SymbolType.LOCAL, + "var4", + String.class.getTypeName(), + 20); + } + + @Test + public void symbolExtraction04() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction04"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction04.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 18, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 3, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 5, 18, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 5); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 5, 18, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + String.class.getTypeName(), + 5); + Scope forLine6Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(forLine6Scope, ScopeType.LOCAL, null, 6, 15, SOURCE_FILE, 1, 1); + assertSymbol( + forLine6Scope.getSymbols().get(0), SymbolType.LOCAL, "i", Integer.TYPE.getTypeName(), 6); + Scope forBodyLine6Scope = forLine6Scope.getScopes().get(0); + assertScope(forBodyLine6Scope, ScopeType.LOCAL, null, 7, 15, SOURCE_FILE, 1, 2); + assertSymbol( + forBodyLine6Scope.getSymbols().get(0), + SymbolType.LOCAL, + "j", + Integer.TYPE.getTypeName(), + 8); + assertSymbol( + forBodyLine6Scope.getSymbols().get(1), + SymbolType.LOCAL, + "var2", + String.class.getTypeName(), + 7); + Scope forBodyLine8Scope = forBodyLine6Scope.getScopes().get(0); + assertScope(forBodyLine8Scope, ScopeType.LOCAL, null, 9, 15, SOURCE_FILE, 1, 2); + assertSymbol( + forBodyLine8Scope.getSymbols().get(0), + SymbolType.LOCAL, + "var3", + String.class.getTypeName(), + 9); + assertSymbol( + forBodyLine8Scope.getSymbols().get(1), + SymbolType.LOCAL, + "var5", + String.class.getTypeName(), + 14); + Scope forLine10Scope = forBodyLine8Scope.getScopes().get(0); + assertScope(forLine10Scope, ScopeType.LOCAL, null, 10, 12, SOURCE_FILE, 1, 1); + assertSymbol( + forLine10Scope.getSymbols().get(0), SymbolType.LOCAL, "k", Integer.TYPE.getTypeName(), 10); + Scope forBodyLine10Scope = forLine10Scope.getScopes().get(0); + assertScope(forBodyLine10Scope, ScopeType.LOCAL, null, 11, 12, SOURCE_FILE, 0, 1); + assertSymbol( + forBodyLine10Scope.getSymbols().get(0), + SymbolType.LOCAL, + "var4", + String.class.getTypeName(), + 11); + } + + @Test + public void symbolExtraction05() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction05"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction05.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 15, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 3, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 5, 15, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 5); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 5, 15, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "i", + Integer.TYPE.getTypeName(), + 5); + Scope whileLine6Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(whileLine6Scope, ScopeType.LOCAL, null, 7, 13, SOURCE_FILE, 1, 2); + assertSymbol( + whileLine6Scope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 7); + assertSymbol( + whileLine6Scope.getSymbols().get(1), SymbolType.LOCAL, "j", Integer.TYPE.getTypeName(), 8); + Scope whileLine9Scope = whileLine6Scope.getScopes().get(0); + assertScope(whileLine9Scope, ScopeType.LOCAL, null, 10, 11, SOURCE_FILE, 0, 1); + assertSymbol( + whileLine9Scope.getSymbols().get(0), + SymbolType.LOCAL, + "var2", + Integer.TYPE.getTypeName(), + 10); + } + + @Test + public void symbolExtraction06() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction06"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction06.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 13, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 3, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 5, 13, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 5); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 5, 13, SOURCE_FILE, 2, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 5); + Scope catchLine9Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(catchLine9Scope, ScopeType.LOCAL, null, 9, 11, SOURCE_FILE, 0, 2); + assertSymbol( + catchLine9Scope.getSymbols().get(0), + SymbolType.LOCAL, + "var3", + Integer.TYPE.getTypeName(), + 10); + assertSymbol( + catchLine9Scope.getSymbols().get(1), + SymbolType.LOCAL, + "rte", + RuntimeException.class.getTypeName(), + 9); + Scope tryLine6Scope = mainMethodLocalScope.getScopes().get(1); + assertScope(tryLine6Scope, ScopeType.LOCAL, null, 7, 8, SOURCE_FILE, 0, 1); + assertSymbol( + tryLine6Scope.getSymbols().get(0), SymbolType.LOCAL, "var2", Integer.TYPE.getTypeName(), 7); + } + + @Test + public void symbolExtraction07() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction07"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction07.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 10, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 3, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 5, 10, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 5); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 5, 10, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "i", + Integer.TYPE.getTypeName(), + 5); + Scope doLine6Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(doLine6Scope, ScopeType.LOCAL, null, 7, 8, SOURCE_FILE, 0, 1); + assertSymbol( + doLine6Scope.getSymbols().get(0), SymbolType.LOCAL, "j", Integer.TYPE.getTypeName(), 7); + } + + @Test + public void symbolExtraction08() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction08"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction08.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 11, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 3, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 5, 11, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 5); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 5, 11, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 5); + Scope line6Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(line6Scope, ScopeType.LOCAL, null, 7, 9, SOURCE_FILE, 0, 2); + assertSymbol( + line6Scope.getSymbols().get(0), SymbolType.LOCAL, "var2", Integer.TYPE.getTypeName(), 7); + assertSymbol( + line6Scope.getSymbols().get(1), SymbolType.LOCAL, "var3", Integer.TYPE.getTypeName(), 8); + } + + @Test + public void symbolExtraction09() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction09"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction09.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 5, 23, SOURCE_FILE, 6, 2); + assertSymbol( + classScope.getSymbols().get(0), + SymbolType.STATIC_FIELD, + "staticIntField", + Integer.TYPE.getTypeName(), + 0); + assertSymbol( + classScope.getSymbols().get(1), + SymbolType.FIELD, + "intField", + Integer.TYPE.getTypeName(), + 0); + assertScope( + classScope.getScopes().get(0), ScopeType.METHOD, "", 5, 17, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 8, 14, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 8); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 8, 14, SOURCE_FILE, 0, 3); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "outside", + Integer.TYPE.getTypeName(), + 8); + assertSymbol( + mainMethodLocalScope.getSymbols().get(1), + SymbolType.LOCAL, + "outside2", + Integer.TYPE.getTypeName(), + 9); + assertSymbol( + mainMethodLocalScope.getSymbols().get(2), + SymbolType.LOCAL, + "lambda", + Supplier.class.getTypeName(), + 10); + Scope processMethodScope = classScope.getScopes().get(2); + assertScope(processMethodScope, ScopeType.METHOD, "process", 19, 23, SOURCE_FILE, 1, 0); + Scope processMethodLocalScope = processMethodScope.getScopes().get(0); + assertScope(processMethodLocalScope, ScopeType.LOCAL, null, 19, 23, SOURCE_FILE, 0, 1); + assertSymbol( + processMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "supplier", + Supplier.class.getTypeName(), + 19); + Scope supplierClosureScope = classScope.getScopes().get(3); + assertScope( + supplierClosureScope, ScopeType.CLOSURE, "lambda$process$1", 20, 21, SOURCE_FILE, 1, 0); + Scope supplierClosureLocalScope = supplierClosureScope.getScopes().get(0); + assertScope(supplierClosureLocalScope, ScopeType.LOCAL, null, 20, 21, SOURCE_FILE, 0, 1); + assertSymbol( + supplierClosureLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 20); + Scope lambdaClosureScope = classScope.getScopes().get(4); + assertScope(lambdaClosureScope, ScopeType.CLOSURE, "lambda$main$0", 11, 12, SOURCE_FILE, 1, 1); + assertSymbol( + lambdaClosureScope.getSymbols().get(0), + SymbolType.ARG, + "outside", + Integer.TYPE.getTypeName(), + 11); + Scope lambdaMethodLocalScope = lambdaClosureScope.getScopes().get(0); + assertScope(lambdaMethodLocalScope, ScopeType.LOCAL, null, 11, 12, SOURCE_FILE, 0, 1); + assertSymbol( + lambdaMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 11); + Scope clinitMethodScope = classScope.getScopes().get(5); + assertScope(clinitMethodScope, ScopeType.METHOD, "", 6, 6, SOURCE_FILE, 0, 0); + } + + @Test + public void symbolExtraction10() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction10"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction10.java"; + when(config.getDebuggerSymbolFlushThreshold()).thenReturn(2); + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + assertEquals(2, symbolSinkMock.jarScopes.get(0).getScopes().size()); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 6, SOURCE_FILE, 2, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 3, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 5, 6, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", String.class.getTypeName(), 5); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 5, 6, SOURCE_FILE, 0, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "winner", + "com.datadog.debugger.symbol.SymbolExtraction10$Inner", + 5); + Scope innerClassScope = symbolSinkMock.jarScopes.get(0).getScopes().get(1); + assertScope(innerClassScope, ScopeType.CLASS, CLASS_NAME + "$Inner", 9, 13, SOURCE_FILE, 2, 1); + assertSymbol( + innerClassScope.getSymbols().get(0), + SymbolType.FIELD, + "field1", + Integer.TYPE.getTypeName(), + 0); + assertScope( + innerClassScope.getScopes().get(0), ScopeType.METHOD, "", 9, 10, SOURCE_FILE, 0, 0); + Scope addToMethod = innerClassScope.getScopes().get(1); + assertScope(addToMethod, ScopeType.METHOD, "addTo", 12, 13, SOURCE_FILE, 1, 1); + assertSymbol( + addToMethod.getSymbols().get(0), SymbolType.ARG, "arg", Integer.TYPE.getTypeName(), 12); + Scope addToMethodLocalScope = addToMethod.getScopes().get(0); + assertScope(addToMethodLocalScope, ScopeType.LOCAL, null, 12, 13, SOURCE_FILE, 0, 1); + assertSymbol( + addToMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 12); + } + + @Test + public void symbolExtraction11() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction11"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction11.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", 1).get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 3, 11, SOURCE_FILE, 2, 1); + assertSymbol( + classScope.getSymbols().get(0), SymbolType.FIELD, "field1", Integer.TYPE.getTypeName(), 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 3, 4, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 6, 11, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", Integer.TYPE.getTypeName(), 6); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 6, 11, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "var1", + Integer.TYPE.getTypeName(), + 6); + Scope ifLine7Scope = mainMethodLocalScope.getScopes().get(0); + assertScope(ifLine7Scope, ScopeType.LOCAL, null, 8, 9, SOURCE_FILE, 0, 1); + assertSymbol( + ifLine7Scope.getSymbols().get(0), SymbolType.LOCAL, "var2", Integer.TYPE.getTypeName(), 8); + } + + @Test + public void symbolExtraction12() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction12"; + final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction12.java"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", 1).get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertScope(classScope, ScopeType.CLASS, CLASS_NAME, 6, 20, SOURCE_FILE, 7, 0); + assertScope(classScope.getScopes().get(0), ScopeType.METHOD, "", 6, 6, SOURCE_FILE, 0, 0); + Scope mainMethodScope = classScope.getScopes().get(1); + assertScope(mainMethodScope, ScopeType.METHOD, "main", 8, 13, SOURCE_FILE, 1, 1); + assertSymbol( + mainMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", Integer.TYPE.getTypeName(), 8); + Scope mainMethodLocalScope = mainMethodScope.getScopes().get(0); + assertScope(mainMethodLocalScope, ScopeType.LOCAL, null, 8, 13, SOURCE_FILE, 0, 2); + assertSymbol( + mainMethodLocalScope.getSymbols().get(0), + SymbolType.LOCAL, + "list", + List.class.getTypeName(), + 8); + assertSymbol( + mainMethodLocalScope.getSymbols().get(1), + SymbolType.LOCAL, + "sum", + Integer.TYPE.getTypeName(), + 12); + Scope fooMethodScope = classScope.getScopes().get(2); + assertScope(fooMethodScope, ScopeType.METHOD, "foo", 17, 20, SOURCE_FILE, 0, 1); + assertSymbol( + fooMethodScope.getSymbols().get(0), SymbolType.ARG, "arg", Integer.TYPE.getTypeName(), 17); + Scope lambdaFoo3MethodScope = classScope.getScopes().get(3); + assertScope( + lambdaFoo3MethodScope, ScopeType.CLOSURE, "lambda$foo$3", 19, 19, SOURCE_FILE, 0, 1); + assertSymbol( + lambdaFoo3MethodScope.getSymbols().get(0), + SymbolType.ARG, + "x", + Integer.TYPE.getTypeName(), + 19); + Scope lambdaFoo2MethodScope = classScope.getScopes().get(4); + assertScope( + lambdaFoo2MethodScope, ScopeType.CLOSURE, "lambda$foo$2", 19, 19, SOURCE_FILE, 0, 1); + assertSymbol( + lambdaFoo2MethodScope.getSymbols().get(0), + SymbolType.ARG, + "x", + Integer.class.getTypeName(), + 19); + Scope lambdaMain1MethodScope = classScope.getScopes().get(5); + assertScope( + lambdaMain1MethodScope, ScopeType.CLOSURE, "lambda$main$1", 11, 11, SOURCE_FILE, 0, 1); + assertSymbol( + lambdaMain1MethodScope.getSymbols().get(0), + SymbolType.ARG, + "x", + Integer.TYPE.getTypeName(), + 11); + Scope lambdaMain0MethodScope = classScope.getScopes().get(6); + assertScope( + lambdaMain0MethodScope, ScopeType.CLOSURE, "lambda$main$0", 11, 11, SOURCE_FILE, 0, 1); + assertSymbol( + lambdaMain0MethodScope.getSymbols().get(0), + SymbolType.ARG, + "x", + Integer.class.getTypeName(), + 11); + } + + @Test + public void symbolExtraction13() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction13"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertLangSpecifics( + classScope.getLanguageSpecifics(), + asList("public"), + asList( + "@com.datadog.debugger.symbol.MyAnnotation", "@com.datadog.debugger.symbol.MyMarker"), + Object.class.getTypeName(), + null, + null); + Scope mainMethodScope = classScope.getScopes().get(1); + assertLangSpecifics( + mainMethodScope.getLanguageSpecifics(), + asList("public", "static"), + asList("@com.datadog.debugger.symbol.MyAnnotation"), + null, + null, + Integer.TYPE.getTypeName()); + assertEquals(3, classScope.getSymbols().size()); + Symbol intField = classScope.getSymbols().get(0); + assertLangSpecifics( + intField.getLanguageSpecifics(), + asList("private"), + asList("@com.datadog.debugger.symbol.MyAnnotation"), + null, + null, + null); + Scope myAnnotationClassScope = symbolSinkMock.jarScopes.get(1).getScopes().get(0); + assertLangSpecifics( + myAnnotationClassScope.getLanguageSpecifics(), + asList("interface", "abstract", "annotation"), + asList("@java.lang.annotation.Target", "@java.lang.annotation.Retention"), + Object.class.getTypeName(), + asList("java.lang.annotation.Annotation"), + null); + Symbol strField = classScope.getSymbols().get(1); + assertLangSpecifics( + strField.getLanguageSpecifics(), + asList("public", "static", "volatile"), + null, + null, + null, + null); + Symbol doubleField = classScope.getSymbols().get(2); + assertLangSpecifics( + doubleField.getLanguageSpecifics(), + asList("protected", "final", "transient"), + null, + null, + null, + null); + } + + @Test + public void symbolExtraction14() throws IOException, URISyntaxException { + final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction14"; + SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); + SymbolExtractionTransformer transformer = + new SymbolExtractionTransformer(symbolSinkMock, config); + instr.addTransformer(transformer); + Class testClass = compileAndLoadClass(CLASS_NAME); + Reflect.on(testClass).call("main", "1").get(); + Scope classScope = symbolSinkMock.jarScopes.get(0).getScopes().get(0); + assertLangSpecifics( + classScope.getLanguageSpecifics(), + asList("public", "abstract"), + null, + Object.class.getTypeName(), + asList("com.datadog.debugger.symbol.I1", "com.datadog.debugger.symbol.I2"), + null); + assertEquals(4, classScope.getScopes().size()); + Scope m1MethodScope = classScope.getScopes().get(2); + assertLangSpecifics( + m1MethodScope.getLanguageSpecifics(), + asList("protected", "abstract"), + null, + null, + null, + Void.TYPE.getTypeName()); + Scope m2MethodScope = classScope.getScopes().get(3); + assertLangSpecifics( + m2MethodScope.getLanguageSpecifics(), + asList("private", "final", "synchronized", "(varargs)", "strictfp"), + null, + null, + null, + String.class.getTypeName()); + Scope i1ClassScope = symbolSinkMock.jarScopes.get(1).getScopes().get(0); + assertLangSpecifics( + i1ClassScope.getLanguageSpecifics(), + asList("interface", "abstract"), + null, + Object.class.getTypeName(), + null, + null); + Scope m3MethodScope = i1ClassScope.getScopes().get(0); + assertLangSpecifics( + m3MethodScope.getLanguageSpecifics(), + asList("public", "default"), + null, + null, + null, + Void.TYPE.getTypeName()); + Scope myEnumClassScope = symbolSinkMock.jarScopes.get(3).getScopes().get(0); + assertLangSpecifics( + myEnumClassScope.getLanguageSpecifics(), + asList("final", "enum"), + null, + Enum.class.getTypeName(), + null, + null); + assertEquals(4, myEnumClassScope.getSymbols().size()); + Symbol oneField = myEnumClassScope.getSymbols().get(0); + assertLangSpecifics( + oneField.getLanguageSpecifics(), + asList("public", "static", "final", "enum"), + null, + null, + null, + null); + Symbol valuesField = myEnumClassScope.getSymbols().get(3); + assertLangSpecifics( + valuesField.getLanguageSpecifics(), + asList("private", "static", "final", "synthetic"), + null, + null, + null, + null); + } + + private void assertLangSpecifics( + LanguageSpecifics languageSpecifics, + List expectedModifiers, + List expectedAnnotations, + String expectedSuperClass, + List expectedInterfaces, + String expectedReturnType) { + if (expectedModifiers == null) { + assertNull(languageSpecifics.getAccessModifiers()); + } else { + assertEquals(expectedModifiers, languageSpecifics.getAccessModifiers()); + } + if (expectedAnnotations == null) { + assertNull(languageSpecifics.getAnnotations()); + } else { + assertEquals(expectedAnnotations, languageSpecifics.getAnnotations()); + } + if (expectedSuperClass == null) { + assertNull(languageSpecifics.getSuperClass()); + } else { + assertEquals(expectedSuperClass, languageSpecifics.getSuperClass()); + } + if (expectedInterfaces == null) { + assertNull(languageSpecifics.getInterfaces()); + } else { + assertEquals(expectedInterfaces, languageSpecifics.getInterfaces()); + } + if (expectedReturnType == null) { + assertNull(languageSpecifics.getReturnType()); + } else { + assertEquals(expectedReturnType, languageSpecifics.getReturnType()); + } + } + + private static void assertScope( + Scope scope, + ScopeType scopeType, + String name, + int startLine, + int endLine, + String sourceFile, + int nbScopes, + int nbSymbols) { + assertEquals(scopeType, scope.getScopeType()); + assertEquals(name, scope.getName()); + assertEquals(startLine, scope.getStartLine()); + assertEquals(endLine, scope.getEndLine()); + assertEquals(sourceFile, scope.getSourceFile()); + assertEquals(nbScopes, scope.getScopes().size()); + assertEquals(nbSymbols, scope.getSymbols().size()); + } + + private void assertSymbol( + Symbol symbol, SymbolType symbolType, String name, String type, int line) { + assertEquals(symbolType, symbol.getSymbolType()); + assertEquals(name, symbol.getName()); + assertEquals(type, symbol.getType()); + assertEquals(line, symbol.getLine()); + } + + static class SymbolSinkMock extends SymbolSink { + final List jarScopes = new ArrayList<>(); + + public SymbolSinkMock(Config config) { + super(config); + } + + @Override + public boolean addScope(Scope jarScope) { + return jarScopes.add(jarScope); + } + } +} diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/uploader/BatchUploaderTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/uploader/BatchUploaderTest.java index e018598f313..99a9e7db46e 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/uploader/BatchUploaderTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/uploader/BatchUploaderTest.java @@ -55,10 +55,9 @@ public void setup() throws IOException { server.start(); url = server.url(URL_PATH); - when(config.getFinalDebuggerSnapshotUrl()).thenReturn(server.url(URL_PATH).toString()); when(config.getDebuggerUploadTimeout()).thenReturn((int) REQUEST_TIMEOUT.getSeconds()); - uploader = new BatchUploader(config, ratelimitedLogger); + uploader = new BatchUploader(config, url.toString(), ratelimitedLogger); } @AfterEach @@ -73,9 +72,7 @@ public void tearDown() throws IOException { @Test void testOkHttpClientForcesCleartextConnspecWhenNotUsingTLS() { - when(config.getFinalDebuggerSnapshotUrl()).thenReturn("http://example.com"); - - uploader = new BatchUploader(config); + uploader = new BatchUploader(config, "http://example.com"); final List connectionSpecs = uploader.getClient().connectionSpecs(); assertEquals(connectionSpecs.size(), 1); @@ -84,9 +81,7 @@ void testOkHttpClientForcesCleartextConnspecWhenNotUsingTLS() { @Test void testOkHttpClientUsesDefaultConnspecsOverTLS() { - when(config.getFinalDebuggerSnapshotUrl()).thenReturn("https://example.com"); - - uploader = new BatchUploader(config); + uploader = new BatchUploader(config, "https://example.com"); final List connectionSpecs = uploader.getClient().connectionSpecs(); assertEquals(connectionSpecs.size(), 2); @@ -163,7 +158,7 @@ public void testTooManyRequests() throws IOException, InterruptedException { // We need to make sure that initial requests that fill up the queue hang to the duration of the // test. So we specify insanely large timeout here. when(config.getDebuggerUploadTimeout()).thenReturn((int) FOREVER_REQUEST_TIMEOUT.getSeconds()); - uploader = new BatchUploader(config); + uploader = new BatchUploader(config, url.toString()); // We have to block all parallel requests to make sure queue is kept full for (int i = 0; i < BatchUploader.MAX_RUNNING_REQUESTS; i++) { @@ -203,15 +198,15 @@ public void testShutdown() throws IOException, InterruptedException { @Test public void testEmptyUrl() { - when(config.getFinalDebuggerSnapshotUrl()).thenReturn(""); - Assertions.assertThrows(IllegalArgumentException.class, () -> new BatchUploader(config)); + Assertions.assertThrows(IllegalArgumentException.class, () -> new BatchUploader(config, "")); } @Test public void testNoContainerId() throws InterruptedException { // we don't explicitly specify a container ID server.enqueue(new MockResponse().setResponseCode(200)); - BatchUploader uploaderWithNoContainerId = new BatchUploader(config, ratelimitedLogger, null); + BatchUploader uploaderWithNoContainerId = + new BatchUploader(config, url.toString(), ratelimitedLogger, null); uploaderWithNoContainerId.upload(SNAPSHOT_BUFFER); uploaderWithNoContainerId.shutdown(); @@ -225,7 +220,7 @@ public void testContainerIdHeader() throws InterruptedException { server.enqueue(new MockResponse().setResponseCode(200)); BatchUploader uploaderWithContainerId = - new BatchUploader(config, ratelimitedLogger, "testContainerId"); + new BatchUploader(config, url.toString(), ratelimitedLogger, "testContainerId"); uploaderWithContainerId.upload(SNAPSHOT_BUFFER); uploaderWithContainerId.shutdown(); @@ -238,7 +233,7 @@ public void testApiKey() throws InterruptedException { server.enqueue(new MockResponse().setResponseCode(200)); when(config.getApiKey()).thenReturn(API_KEY_VALUE); - BatchUploader uploaderWithApiKey = new BatchUploader(config, ratelimitedLogger); + BatchUploader uploaderWithApiKey = new BatchUploader(config, url.toString(), ratelimitedLogger); uploaderWithApiKey.upload(SNAPSHOT_BUFFER); uploaderWithApiKey.shutdown(); diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction01.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction01.java new file mode 100644 index 00000000000..aea27733512 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction01.java @@ -0,0 +1,22 @@ +package com.datadog.debugger.symbol; +public class SymbolExtraction01 { + public static int main(String arg) { + int var1 = 1; + if (Integer.parseInt(arg) == 2) { + int var2 = 2; + for (int i = 0; i <= 9; i++) { + int foo = 13; + int bar = 13; + System.out.println(i + foo + bar); + int j = 0; + while (j < 10) { + int var4 = 1; + j++; + } + } + return var2; + } + int var3 = 3; + return var1 + var3; + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction02.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction02.java new file mode 100644 index 00000000000..2b0396c23a5 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction02.java @@ -0,0 +1,8 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction02 { + public static int main(String arg) { + String var1 = "var1"; + return var1.length(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction03.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction03.java new file mode 100644 index 00000000000..bc2be93c149 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction03.java @@ -0,0 +1,30 @@ + +package com.datadog.debugger.symbol; + +public class SymbolExtraction03 { + public static int main(String arg) { + String var1 = "var1"; + if (arg.equals("foo")) { + String var2 = "var2"; + System.out.println(var2); + } else { + System.out.println(var1); + String var31 = "var31"; + String var32 = "var32"; + System.out.println(var1); + String var30 = "var30"; + System.out.println(var1); + String var3 = "var3"; + System.out.println(var3); + if (arg.equals(var3)) { + String var4 = "var4"; + System.out.println(var4); + } + if (arg.equals(var1)) { + return var3.length(); + } + } + String var5 = "var5"; + return var1.length(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction04.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction04.java new file mode 100644 index 00000000000..f667317cb92 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction04.java @@ -0,0 +1,20 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction04 { + public static int main(String arg) { + String var1 = "var1"; + for (int i = 0; i < 10; i++) { + String var2 = "var2"; + for (int j = 0; j < 10; j++) { + String var3 = "var3"; + for (int k = 0; k < 10; k++) { + String var4 = "var4"; + System.out.println("var4 = " + var4); + } + String var5 = "var5"; + System.out.println("var5 = " + var5); + } + } + return var1.length(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction05.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction05.java new file mode 100644 index 00000000000..327adb10431 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction05.java @@ -0,0 +1,17 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction05 { + public static int main(String arg) { + int i = 0; + while (i < 10) { + int var1 = 10; + int j = 0; + while (j < 10) { + int var2 = 1; + j++; + } + i++; + } + return arg.length(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction06.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction06.java new file mode 100644 index 00000000000..4eeb3173c81 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction06.java @@ -0,0 +1,15 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction06 { + public static int main(String arg) { + int var1 = 1; + try { + int var2 = 2; + throw new RuntimeException("" + var1); + } catch (RuntimeException rte) { + int var3 = 3; + System.out.println("rte = " + rte.getMessage() + var3); + } + return arg.length(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction07.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction07.java new file mode 100644 index 00000000000..6b3b6f04af1 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction07.java @@ -0,0 +1,12 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction07 { + public static int main(String arg) { + int i = 10; + do { + int j = i + 12; + i--; + } while (i > 0); + return arg.length(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction08.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction08.java new file mode 100644 index 00000000000..a06a4561a64 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction08.java @@ -0,0 +1,13 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction08 { + public static int main(String arg) { + int var1 = 1; + { + int var2 = 2; + int var3 = 3; + int var4 = var2 + var3; // var4 is not in the LocalVariableTable because last statement of the scope + } + return arg.length(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction09.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction09.java new file mode 100644 index 00000000000..fac032d858c --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction09.java @@ -0,0 +1,25 @@ +package com.datadog.debugger.symbol; + +import java.util.function.Supplier; + +public class SymbolExtraction09 { + static int staticIntField = 42; + public static int main(String arg) { + int outside = 12; + int outside2 = 1337; + Supplier lambda = () -> { + int var1 = 1; + return var1 + outside + staticIntField; + }; + return lambda.get(); + } + + int intField = 42; + public int process() { + Supplier supplier = () -> { + int var1 = 1; + return var1 + intField; + }; + return supplier.get(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction10.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction10.java new file mode 100644 index 00000000000..b1d6f28bea7 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction10.java @@ -0,0 +1,16 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction10 { + public static int main(String arg) { + Inner winner = new Inner(); + return winner.addTo(12); + } + + static class Inner { + private final int field1 = 1; + public int addTo(int arg) { + int var1 = 2; + return var1 + arg; + } + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction11.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction11.java new file mode 100644 index 00000000000..b4ba9ccd520 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction11.java @@ -0,0 +1,13 @@ +package com.datadog.debugger.symbol; + +public class SymbolExtraction11 { + private final int field1 = 1; + public static int main(int arg) { + int var1 = 1; + if (arg == 42) { + int var2 = 2; + return var2; + } + return var1 + arg; + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction12.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction12.java new file mode 100644 index 00000000000..330fbb1ae81 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction12.java @@ -0,0 +1,22 @@ +package com.datadog.debugger.symbol; + +import java.util.Arrays; +import java.util.List; + +public class SymbolExtraction12 { + public static int main(int arg) { + List list = Arrays.asList(1, 2, 3); + int sum = list + .stream() + .mapToInt(x -> x + 1).map(x -> x - 12) + .sum(); + return sum; + } + + public static int foo(int arg) { + return Arrays.asList(1, 2, 3, 4) + .stream() + .mapToInt(x -> x + 1).map(x -> x - 12) + .sum(); + } +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction13.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction13.java new file mode 100644 index 00000000000..6cd268066ef --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction13.java @@ -0,0 +1,39 @@ +package com.datadog.debugger.symbol; + +import org.junit.jupiter.api.Test; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@MyAnnotation("class") +@MyMarker +public class SymbolExtraction13 { + + @MyAnnotation("field") + private int intField; + public static volatile String strField; + protected final transient double doubleField = 3.14; + + @MyAnnotation("method") + public static int main(String arg) { + System.out.println(MyAnnotation.class); + return 42; + } + + private static class InnerClass { + + } +} + +@Target({ElementType.TYPE, ElementType.METHOD, ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +@interface MyAnnotation { + String value() default ""; +} + +@Target({ElementType.TYPE, ElementType.METHOD, ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +@interface MyMarker { +} diff --git a/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction14.java b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction14.java new file mode 100644 index 00000000000..417867c3659 --- /dev/null +++ b/dd-java-agent/agent-debugger/src/test/resources/com/datadog/debugger/symbol/SymbolExtraction14.java @@ -0,0 +1,36 @@ +package com.datadog.debugger.symbol; + +import org.junit.jupiter.api.Test; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +public abstract class SymbolExtraction14 extends Object implements I1, I2{ + + public static int main(String arg) { + System.out.println(MyEnum.ONE); + return 42; + } + + protected abstract void m1(); + private strictfp synchronized final String m2(String... strVarArgs) { + return null; + } + +} + +interface I1 { + default void m3(){} +} + +interface I2 { + +} + +enum MyEnum { + ONE, + TWO, + THREE +} diff --git a/dd-java-agent/agent-iast/build.gradle b/dd-java-agent/agent-iast/build.gradle index 3a42e0871f4..068bbf5b61d 100644 --- a/dd-java-agent/agent-iast/build.gradle +++ b/dd-java-agent/agent-iast/build.gradle @@ -1,13 +1,29 @@ +import net.ltgt.gradle.errorprone.CheckSeverity + plugins { id 'com.github.johnrengelman.shadow' id 'me.champeau.jmh' id 'java-test-fixtures' id 'com.google.protobuf' version '0.8.18' + id 'net.ltgt.errorprone' version '3.1.0' } apply from: "$rootDir/gradle/java.gradle" apply from: "$rootDir/gradle/version.gradle" +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(11)) + } + sourceCompatibility = 1.8 + targetCompatibility = 1.8 +} + +tasks.withType(AbstractCompile).configureEach { + // ensure no APIs beyond JDK8 are used + options.compilerArgs.addAll(['--release', '8']) +} + // First version with Mac M1 support def grpcVersion = '1.42.2' protobuf { @@ -49,6 +65,10 @@ dependencies { jmh project(':dd-java-agent:agent-builder') jmh project(':dd-java-agent:instrumentation:iast-instrumenter') jmh project(':dd-java-agent:instrumentation:java-lang') + + compileOnly('org.jetbrains:annotations:24.0.0') + errorprone('com.uber.nullaway:nullaway:0.10.15') + errorprone('com.google.errorprone:error_prone_core:2.23.0') } shadowJar { @@ -80,7 +100,6 @@ ext { tasks.withType(Test).configureEach { jvmArgs += ['-Ddd.iast.enabled=true'] } -def rootDir = project.rootDir spotless { java { target 'src/**/*.java' @@ -104,3 +123,16 @@ sourceSets { } } } + +tasks.withType(JavaCompile).configureEach { + if (name == 'compileJava') { + options.errorprone { + check("NullAway", CheckSeverity.ERROR) + option("NullAway:AnnotatedPackages", "com.datadog.iast") + disableAllWarnings = true // only errors for now + } + } else { + // disable null away for test and jmh + options.errorprone.enabled = false + } +} diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderAppendBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderAppendBenchmark.java index babe608c0f3..e4bb2ee9f46 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderAppendBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderAppendBenchmark.java @@ -1,6 +1,6 @@ package com.datadog.iast.propagation; -import static com.datadog.iast.model.Range.NOT_MARKED; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderBatchBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderBatchBenchmark.java index 1d5f61a43bd..8bb5d4adb20 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderBatchBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderBatchBenchmark.java @@ -1,5 +1,6 @@ package com.datadog.iast.propagation; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import static java.util.concurrent.TimeUnit.MICROSECONDS; import com.datadog.iast.IastRequestContext; @@ -34,8 +35,7 @@ protected StringBuilderBatchBenchmark.Context initializeContext() { final String value; if (current < limit) { value = - tainted( - context, UUID.randomUUID().toString(), new Range(3, 6, source(), Range.NOT_MARKED)); + tainted(context, UUID.randomUUID().toString(), new Range(3, 6, source(), NOT_MARKED)); } else { value = notTainted(UUID.randomUUID().toString()); } diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderInitBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderInitBenchmark.java index a3c832355dc..ec11b927ff2 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderInitBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderInitBenchmark.java @@ -1,5 +1,7 @@ package com.datadog.iast.propagation; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; + import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; import datadog.trace.instrumentation.java.lang.StringBuilderCallSite; @@ -14,7 +16,7 @@ protected Context initializeContext() { final IastRequestContext context = new IastRequestContext(); final String notTainted = notTainted("I am not a tainted string"); final String tainted = - tainted(context, "I am a tainted string", new Range(3, 6, source(), Range.NOT_MARKED)); + tainted(context, "I am a tainted string", new Range(3, 6, source(), NOT_MARKED)); return new Context(context, notTainted, tainted); } diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderToStringBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderToStringBenchmark.java index 707e9b04611..1a8468610a4 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderToStringBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringBuilderToStringBenchmark.java @@ -1,5 +1,7 @@ package com.datadog.iast.propagation; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; + import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; import datadog.trace.instrumentation.java.lang.StringBuilderCallSite; @@ -18,7 +20,7 @@ protected Context initializeContext() { tainted( context, new StringBuilder("I am a tainted string builder"), - new Range(5, 7, source(), Range.NOT_MARKED)); + new Range(5, 7, source(), NOT_MARKED)); return new Context(context, notTaintedBuilder, taintedBuilder); } diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatBenchmark.java index 4dded1b999e..0858263ff00 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatBenchmark.java @@ -1,5 +1,7 @@ package com.datadog.iast.propagation; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; + import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; import datadog.trace.instrumentation.java.lang.StringCallSite; @@ -13,7 +15,7 @@ protected StringConcatBenchmark.Context initializeContext() { final IastRequestContext context = new IastRequestContext(); final String notTainted = notTainted("I am not a tainted string"); final String tainted = - tainted(context, "I am a tainted string", new Range(3, 5, source(), Range.NOT_MARKED)); + tainted(context, "I am a tainted string", new Range(3, 5, source(), NOT_MARKED)); return new StringConcatBenchmark.Context(context, notTainted, tainted); } diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBatchBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBatchBenchmark.java index 42e1741f361..59f0353ec51 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBatchBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBatchBenchmark.java @@ -1,5 +1,6 @@ package com.datadog.iast.propagation; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import static java.util.concurrent.TimeUnit.MICROSECONDS; import com.datadog.iast.IastRequestContext; @@ -54,7 +55,7 @@ protected StringConcatFactoryBatchBenchmark.Context initializeContext() { double current = i / (double) stringCount; final String value; if (current < limit) { - value = tainted(context, "Yep, tainted", new Range(3, 5, source(), Range.NOT_MARKED)); + value = tainted(context, "Yep, tainted", new Range(3, 5, source(), NOT_MARKED)); } else { value = notTainted("Nop, tainted"); } diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBenchmark.java index 6f99576fa56..9008bea3ad4 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringConcatFactoryBenchmark.java @@ -1,5 +1,7 @@ package com.datadog.iast.propagation; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; + import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; import datadog.trace.api.iast.InstrumentationBridge; @@ -13,8 +15,7 @@ public class StringConcatFactoryBenchmark protected StringConcatFactoryBenchmark.Context initializeContext() { final IastRequestContext context = new IastRequestContext(); final String notTainted = notTainted("Nop, tainted"); - final String tainted = - tainted(context, "Yep, tainted", new Range(3, 5, source(), Range.NOT_MARKED)); + final String tainted = tainted(context, "Yep, tainted", new Range(3, 5, source(), NOT_MARKED)); return new StringConcatFactoryBenchmark.Context(context, notTainted, tainted); } diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringJoinBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringJoinBenchmark.java index 5b7b4d87a69..3f8bb4b6cfd 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringJoinBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringJoinBenchmark.java @@ -1,6 +1,6 @@ package com.datadog.iast.propagation; -import static com.datadog.iast.model.Range.NOT_MARKED; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; diff --git a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringSubsequenceBenchmark.java b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringSubsequenceBenchmark.java index 78e8f030be8..c8ba76a2add 100644 --- a/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringSubsequenceBenchmark.java +++ b/dd-java-agent/agent-iast/src/jmh/java/com/datadog/iast/propagation/StringSubsequenceBenchmark.java @@ -1,6 +1,6 @@ package com.datadog.iast.propagation; -import static com.datadog.iast.model.Range.NOT_MARKED; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Dependencies.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Dependencies.java new file mode 100644 index 00000000000..e4df7602205 --- /dev/null +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Dependencies.java @@ -0,0 +1,41 @@ +package com.datadog.iast; + +import com.datadog.iast.overhead.OverheadController; +import datadog.trace.api.Config; +import datadog.trace.util.stacktrace.StackWalker; +import javax.annotation.Nonnull; + +public class Dependencies { + + private final Config config; + private final Reporter reporter; + private final OverheadController overheadController; + private final StackWalker stackWalker; + + public Dependencies( + @Nonnull final Config config, + @Nonnull final Reporter reporter, + @Nonnull final OverheadController overheadController, + @Nonnull final StackWalker stackWalker) { + this.config = config; + this.reporter = reporter; + this.overheadController = overheadController; + this.stackWalker = stackWalker; + } + + public Config getConfig() { + return config; + } + + public Reporter getReporter() { + return reporter; + } + + public OverheadController getOverheadController() { + return overheadController; + } + + public StackWalker getStackWalker() { + return stackWalker; + } +} diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/GrpcRequestMessageHandler.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/GrpcRequestMessageHandler.java index 7091ffc23af..8c167fbed7b 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/GrpcRequestMessageHandler.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/GrpcRequestMessageHandler.java @@ -2,7 +2,7 @@ import datadog.trace.api.gateway.Flow; import datadog.trace.api.gateway.RequestContext; -import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.SourceTypes; import datadog.trace.api.iast.propagation.PropagationModule; @@ -29,9 +29,9 @@ public class GrpcRequestMessageHandler implements BiFunction apply(final RequestContext ctx, final Object o) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null && o != null) { - final IastRequestContext iastCtx = ctx.getData(RequestContextSlot.IAST); + final IastContext iastCtx = IastContext.Provider.get(ctx); module.taintDeeply( - iastCtx, SourceTypes.GRPC_BODY, o, GrpcRequestMessageHandler::isProtobufArtifact); + iastCtx, o, SourceTypes.GRPC_BODY, GrpcRequestMessageHandler::isProtobufArtifact); } return Flow.ResultFlow.empty(); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/HasDependencies.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/HasDependencies.java deleted file mode 100644 index 9e1d7cf758f..00000000000 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/HasDependencies.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.datadog.iast; - -import com.datadog.iast.overhead.OverheadController; -import datadog.trace.api.Config; -import datadog.trace.util.stacktrace.StackWalker; -import javax.annotation.Nonnull; - -public interface HasDependencies { - - void registerDependencies(@Nonnull Dependencies dependencies); - - class Dependencies { - private final Config config; - private final Reporter reporter; - private final OverheadController overheadController; - private final StackWalker stackWalker; - - public Dependencies( - @Nonnull final Config config, - @Nonnull final Reporter reporter, - @Nonnull final OverheadController overheadController, - @Nonnull final StackWalker stackWalker) { - this.config = config; - this.reporter = reporter; - this.overheadController = overheadController; - this.stackWalker = stackWalker; - } - - public Config getConfig() { - return config; - } - - public Reporter getReporter() { - return reporter; - } - - public OverheadController getOverheadController() { - return overheadController; - } - - public StackWalker getStackWalker() { - return stackWalker; - } - } -} diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastRequestContext.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastRequestContext.java index a5472fb3b00..15817bde263 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastRequestContext.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastRequestContext.java @@ -4,25 +4,25 @@ import com.datadog.iast.overhead.OverheadContext; import com.datadog.iast.taint.TaintedObjects; import datadog.trace.api.gateway.RequestContext; -import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.telemetry.IastMetricCollector; import datadog.trace.api.iast.telemetry.IastMetricCollector.HasMetricCollector; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; import javax.annotation.Nullable; -public class IastRequestContext implements HasMetricCollector { +public class IastRequestContext implements IastContext, HasMetricCollector { private final VulnerabilityBatch vulnerabilityBatch; private final AtomicBoolean spanDataIsSet; private final TaintedObjects taintedObjects; private final OverheadContext overheadContext; - private final IastMetricCollector collector; - private volatile String strictTransportSecurity; - private volatile String xContentTypeOptions; - private volatile String xForwardedProto; - private volatile String contentType; + @Nullable private final IastMetricCollector collector; + @Nullable private volatile String strictTransportSecurity; + @Nullable private volatile String xContentTypeOptions; + @Nullable private volatile String xForwardedProto; + @Nullable private volatile String contentType; public IastRequestContext() { this(TaintedObjects.acquire(), null); @@ -33,7 +33,7 @@ public IastRequestContext(final TaintedObjects taintedObjects) { } public IastRequestContext( - final TaintedObjects taintedObjects, final IastMetricCollector collector) { + final TaintedObjects taintedObjects, @Nullable final IastMetricCollector collector) { this.vulnerabilityBatch = new VulnerabilityBatch(); this.spanDataIsSet = new AtomicBoolean(false); this.overheadContext = new OverheadContext(); @@ -45,6 +45,7 @@ public VulnerabilityBatch getVulnerabilityBatch() { return vulnerabilityBatch; } + @Nullable public String getStrictTransportSecurity() { return strictTransportSecurity; } @@ -53,6 +54,7 @@ public void setStrictTransportSecurity(final String strictTransportSecurity) { this.strictTransportSecurity = strictTransportSecurity; } + @Nullable public String getxContentTypeOptions() { return xContentTypeOptions; } @@ -61,6 +63,7 @@ public void setxContentTypeOptions(final String xContentTypeOptions) { this.xContentTypeOptions = xContentTypeOptions; } + @Nullable public String getxForwardedProto() { return xForwardedProto; } @@ -69,6 +72,7 @@ public void setxForwardedProto(final String xForwardedProto) { this.xForwardedProto = xForwardedProto; } + @Nullable public String getContentType() { return contentType; } @@ -85,6 +89,7 @@ public OverheadContext getOverheadContext() { return overheadContext; } + @Nonnull public TaintedObjects getTaintedObjects() { return taintedObjects; } @@ -97,22 +102,21 @@ public IastMetricCollector getMetricCollector() { @Nullable public static IastRequestContext get() { - return get(AgentTracer.activeSpan()); + return asRequestContext(IastContext.Provider.get()); } @Nullable public static IastRequestContext get(final AgentSpan span) { - if (span == null) { - return null; - } - return get(span.getRequestContext()); + return asRequestContext(IastContext.Provider.get(span)); } @Nullable public static IastRequestContext get(final RequestContext reqCtx) { - if (reqCtx == null) { - return null; - } - return reqCtx.getData(RequestContextSlot.IAST); + return asRequestContext(IastContext.Provider.get(reqCtx)); + } + + @Nullable + private static IastRequestContext asRequestContext(final IastContext ctx) { + return ctx instanceof IastRequestContext ? (IastRequestContext) ctx : null; } } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastSystem.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastSystem.java index b5c3e7131f4..5da60ff4d30 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastSystem.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/IastSystem.java @@ -1,6 +1,5 @@ package com.datadog.iast; -import com.datadog.iast.HasDependencies.Dependencies; import com.datadog.iast.overhead.OverheadController; import com.datadog.iast.propagation.FastCodecModule; import com.datadog.iast.propagation.PropagationModuleImpl; @@ -15,6 +14,7 @@ import com.datadog.iast.sink.PathTraversalModuleImpl; import com.datadog.iast.sink.SqlInjectionModuleImpl; import com.datadog.iast.sink.SsrfModuleImpl; +import com.datadog.iast.sink.StacktraceLeakModuleImpl; import com.datadog.iast.sink.TrustBoundaryViolationModuleImpl; import com.datadog.iast.sink.UnvalidatedRedirectModuleImpl; import com.datadog.iast.sink.WeakCipherModuleImpl; @@ -23,7 +23,6 @@ import com.datadog.iast.sink.XContentTypeModuleImpl; import com.datadog.iast.sink.XPathInjectionModuleImpl; import com.datadog.iast.sink.XssModuleImpl; -import com.datadog.iast.source.WebModuleImpl; import com.datadog.iast.telemetry.TelemetryRequestEndedHandler; import com.datadog.iast.telemetry.TelemetryRequestStartedHandler; import datadog.trace.api.Config; @@ -41,9 +40,9 @@ import datadog.trace.util.AgentTaskScheduler; import datadog.trace.util.stacktrace.StackWalkerFactory; import java.util.function.BiFunction; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +55,8 @@ public static void start(final SubscriptionService ss) { start(ss, null); } - public static void start(final SubscriptionService ss, OverheadController overheadController) { + public static void start( + final SubscriptionService ss, @Nullable OverheadController overheadController) { final Config config = Config.get(); if (config.getIastActivation() != ProductActivation.FULLY_ENABLED) { LOGGER.debug("IAST is disabled"); @@ -71,7 +71,7 @@ public static void start(final SubscriptionService ss, OverheadController overhe final Dependencies dependencies = new Dependencies(config, reporter, overheadController, StackWalkerFactory.INSTANCE); final boolean addTelemetry = config.getIastTelemetryVerbosity() != Verbosity.OFF; - iastModules().forEach(registerModule(dependencies)); + iastModules(dependencies).forEach(InstrumentationBridge::registerIastModule); registerRequestStartedCallback(ss, addTelemetry, dependencies); registerRequestEndedCallback(ss, addTelemetry, dependencies); registerHeadersCallback(ss); @@ -79,39 +79,30 @@ public static void start(final SubscriptionService ss, OverheadController overhe LOGGER.debug("IAST started"); } - private static Consumer registerModule(final Dependencies dependencies) { - return module -> { - if (module instanceof HasDependencies) { - ((HasDependencies) module).registerDependencies(dependencies); - } - InstrumentationBridge.registerIastModule(module); - }; - } - - private static Stream iastModules() { + private static Stream iastModules(final Dependencies dependencies) { return Stream.of( - new WebModuleImpl(), new StringModuleImpl(), new FastCodecModule(), - new SqlInjectionModuleImpl(), - new PathTraversalModuleImpl(), - new CommandInjectionModuleImpl(), - new WeakCipherModuleImpl(), - new WeakHashModuleImpl(), - new LdapInjectionModuleImpl(), + new SqlInjectionModuleImpl(dependencies), + new PathTraversalModuleImpl(dependencies), + new CommandInjectionModuleImpl(dependencies), + new WeakCipherModuleImpl(dependencies), + new WeakHashModuleImpl(dependencies), + new LdapInjectionModuleImpl(dependencies), new PropagationModuleImpl(), - new HttpResponseHeaderModuleImpl(), - new HstsMissingHeaderModuleImpl(), + new HttpResponseHeaderModuleImpl(dependencies), + new HstsMissingHeaderModuleImpl(dependencies), new InsecureCookieModuleImpl(), new NoHttpOnlyCookieModuleImpl(), - new XContentTypeModuleImpl(), + new XContentTypeModuleImpl(dependencies), new NoSameSiteCookieModuleImpl(), - new SsrfModuleImpl(), - new UnvalidatedRedirectModuleImpl(), - new WeakRandomnessModuleImpl(), - new XPathInjectionModuleImpl(), - new TrustBoundaryViolationModuleImpl(), - new XssModuleImpl()); + new SsrfModuleImpl(dependencies), + new UnvalidatedRedirectModuleImpl(dependencies), + new WeakRandomnessModuleImpl(dependencies), + new XPathInjectionModuleImpl(dependencies), + new TrustBoundaryViolationModuleImpl(dependencies), + new XssModuleImpl(dependencies), + new StacktraceLeakModuleImpl(dependencies)); } private static void registerRequestStartedCallback( diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Reporter.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Reporter.java index e51cf258b38..bb4f3436a27 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Reporter.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Reporter.java @@ -30,7 +30,7 @@ public class Reporter { this(Config.get(), null); } - public Reporter(final Config config, final AgentTaskScheduler taskScheduler) { + public Reporter(final Config config, @Nullable final AgentTaskScheduler taskScheduler) { this( config.isIastDeduplicationEnabled() ? new HashBasedDeduplication(taskScheduler) @@ -108,11 +108,11 @@ protected static class HashBasedDeduplication implements Predicate hashes; - public HashBasedDeduplication(final AgentTaskScheduler taskScheduler) { + public HashBasedDeduplication(@Nullable final AgentTaskScheduler taskScheduler) { this(DEFAULT_MAX_SIZE, taskScheduler); } - HashBasedDeduplication(final int size, final AgentTaskScheduler taskScheduler) { + HashBasedDeduplication(final int size, @Nullable final AgentTaskScheduler taskScheduler) { maxSize = size; hashes = ConcurrentHashMap.newKeySet(size); if (taskScheduler != null) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestEndedHandler.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestEndedHandler.java index 25611c79643..fc2c9a18975 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestEndedHandler.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestEndedHandler.java @@ -3,7 +3,6 @@ import static com.datadog.iast.IastTag.ANALYZED; import static com.datadog.iast.IastTag.SKIPPED; -import com.datadog.iast.HasDependencies.Dependencies; import com.datadog.iast.overhead.OverheadController; import com.datadog.iast.taint.TaintedObjects; import datadog.trace.api.gateway.Flow; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestStartedHandler.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestStartedHandler.java index 9c680717743..4d5c851f52c 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestStartedHandler.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/RequestStartedHandler.java @@ -1,6 +1,5 @@ package com.datadog.iast; -import com.datadog.iast.HasDependencies.Dependencies; import com.datadog.iast.overhead.OverheadController; import datadog.trace.api.gateway.Flow; import java.util.function.Supplier; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Evidence.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Evidence.java index 65b2f78e374..c4dc17d27e5 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Evidence.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Evidence.java @@ -17,6 +17,8 @@ public final class Evidence { private final transient @Nonnull Context context = new Evidence.Context(4); /** For deserialization in tests via moshi */ + @Deprecated + @SuppressWarnings({"NullAway", "DataFlowIssue", "unused"}) private Evidence() { this(null, null); } @@ -25,15 +27,17 @@ public Evidence(final String value) { this(value, null); } - public Evidence(final String value, final Range[] ranges) { + public Evidence(@Nonnull final String value, @Nullable final Range[] ranges) { this.value = value; this.ranges = ranges; } + @Nonnull public String getValue() { return value; } + @Nullable public Range[] getRanges() { return ranges; } @@ -76,6 +80,7 @@ public boolean put(final String key, final Object value) { return true; } + @Nullable @SuppressWarnings("unchecked") public E get(@Nonnull final String key) { return (E) context.get(key); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Location.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Location.java index 2522699121b..1dbe22b545d 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Location.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Location.java @@ -1,25 +1,26 @@ package com.datadog.iast.model; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import javax.annotation.Nullable; public final class Location { - private final String path; + @Nullable private final String path; private final int line; - private final String method; + @Nullable private final String method; - private Long spanId; + @Nullable private Long spanId; - private transient String serviceName; + @Nullable private transient String serviceName; private Location( - final Long spanId, - final String path, + @Nullable final Long spanId, + @Nullable final String path, final int line, - final String method, - final String serviceName) { + @Nullable final String method, + @Nullable final String serviceName) { this.spanId = spanId; this.path = path; this.line = line; @@ -27,7 +28,8 @@ private Location( this.serviceName = serviceName; } - public static Location forSpanAndStack(final AgentSpan span, final StackTraceElement stack) { + public static Location forSpanAndStack( + @Nullable final AgentSpan span, final StackTraceElement stack) { return new Location( spanId(span), stack.getClassName(), @@ -54,6 +56,7 @@ public long getSpanId() { return spanId == null ? 0 : spanId; } + @Nullable public String getPath() { return path; } @@ -62,26 +65,30 @@ public int getLine() { return line; } + @Nullable public String getMethod() { return method; } + @Nullable public String getServiceName() { return serviceName; } - public void updateSpan(final AgentSpan span) { + public void updateSpan(@Nullable final AgentSpan span) { if (span != null) { this.spanId = span.getSpanId(); this.serviceName = span.getServiceName(); } } - private static Long spanId(AgentSpan span) { + @Nullable + private static Long spanId(@Nullable AgentSpan span) { return span != null ? span.getSpanId() : null; } - private static String serviceName(AgentSpan span) { + @Nullable + private static String serviceName(@Nullable AgentSpan span) { return span != null ? span.getServiceName() : null; } } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Range.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Range.java index 97caac40bd3..a8e1d43b28a 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Range.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Range.java @@ -1,5 +1,7 @@ package com.datadog.iast.model; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; + import com.datadog.iast.model.json.SourceIndex; import com.datadog.iast.util.Ranged; import java.util.Objects; @@ -9,8 +11,6 @@ public final class Range implements Ranged { - public static final int NOT_MARKED = 0; - private final @Nonnegative int start; private final @Nonnegative int length; private final @Nonnull @SourceIndex Source source; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Source.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Source.java index 6dc5956adc7..31acd483a7a 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Source.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Source.java @@ -5,13 +5,19 @@ import datadog.trace.api.iast.Taintable; import java.util.Objects; import java.util.StringJoiner; +import javax.annotation.Nullable; public final class Source implements Taintable.Source { private final @SourceTypeString byte origin; - private final String name; - private final String value; + @Nullable private final String name; + @Nullable private final String value; - public Source(final byte origin, final String name, final String value) { + public Source( + final byte origin, @Nullable final CharSequence name, @Nullable final CharSequence value) { + this(origin, name == null ? null : name.toString(), value == null ? null : value.toString()); + } + + public Source(final byte origin, @Nullable final String name, @Nullable final String value) { this.origin = origin; this.name = name; this.value = value; @@ -23,11 +29,13 @@ public byte getOrigin() { } @Override + @Nullable public String getName() { return name; } @Override + @Nullable public String getValue() { return value; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Vulnerability.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Vulnerability.java index 1e119321ff1..97ecf815775 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Vulnerability.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/Vulnerability.java @@ -15,21 +15,26 @@ public final class Vulnerability { private long hash; public Vulnerability( - final VulnerabilityType type, final Location location, final Evidence evidence) { + @Nonnull final VulnerabilityType type, + @Nonnull final Location location, + @Nullable final Evidence evidence) { this.type = type; this.location = location; this.evidence = evidence; this.hash = type.calculateHash(this); } + @Nonnull public VulnerabilityType getType() { return type; } + @Nonnull public Location getLocation() { return location; } + @Nullable public Evidence getEvidence() { return evidence; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityBatch.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityBatch.java index 17f59fbdd7c..fa5193b0a00 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityBatch.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityBatch.java @@ -3,12 +3,13 @@ import com.datadog.iast.model.json.VulnerabilityEncoding; import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; /** Collects vulnerabilities and serializes to JSON lazily on {@link #toString()}. */ public final class VulnerabilityBatch { - private List vulnerabilities; - private volatile String json; + @Nullable private List vulnerabilities; + @Nullable private volatile String json; public void add(final Vulnerability v) { synchronized (this) { @@ -21,6 +22,7 @@ public void add(final Vulnerability v) { } /** Internal list of vulnerabilities. Not thread-safe. */ + @Nullable public List getVulnerabilities() { return vulnerabilities; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityType.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityType.java index deb7888adaf..62fdc6db56e 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityType.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/VulnerabilityType.java @@ -1,10 +1,11 @@ package com.datadog.iast.model; -import static com.datadog.iast.model.Range.NOT_MARKED; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import datadog.trace.api.iast.VulnerabilityMarks; import datadog.trace.api.iast.VulnerabilityTypes; import java.io.File; +import java.nio.charset.StandardCharsets; import java.util.zip.CRC32; import javax.annotation.Nonnull; @@ -64,6 +65,9 @@ public interface VulnerabilityType { InjectionType XSS = new InjectionTypeImpl(VulnerabilityTypes.XSS_STRING, VulnerabilityMarks.XSS_MARK, ' '); + VulnerabilityType STACKTRACE_LEAK = + new VulnerabilityTypeImpl(VulnerabilityTypes.STACKTRACE_LEAK_STRING, NOT_MARKED); + String name(); /** A bit flag to ignore tainted ranges for this vulnerability. Set to 0 if none. */ @@ -99,17 +103,24 @@ public int mark() { @Override public long calculateHash(@Nonnull final Vulnerability vulnerability) { CRC32 crc = new CRC32(); - crc.update(name().getBytes()); + update(crc, name()); final Location location = vulnerability.getLocation(); if (location != null) { crc.update(location.getLine()); - crc.update(location.getPath().getBytes()); + if (location.getPath() != null) { + update(crc, location.getPath()); + } if (location.getLine() <= -1 && location.getMethod() != null) { - crc.update(location.getMethod().getBytes()); + update(crc, location.getMethod()); } } return crc.getValue(); } + + protected void update(final CRC32 crc, final String value) { + final byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + crc.update(bytes, 0, bytes.length); + } } class InjectionTypeImpl extends VulnerabilityTypeImpl implements InjectionType { @@ -135,10 +146,10 @@ public HeaderVulnerabilityType(@Nonnull String name, int vulnerabilityMark) { @Override public long calculateHash(@Nonnull final Vulnerability vulnerability) { CRC32 crc = new CRC32(); - crc.update(name().getBytes()); + update(crc, name()); String serviceName = vulnerability.getLocation().getServiceName(); if (serviceName != null) { - crc.update(serviceName.getBytes()); + update(crc, serviceName); } return crc.getValue(); } @@ -152,10 +163,10 @@ public CookieVulnerabilityType(@Nonnull String name, int vulnerabilityMark) { @Override public long calculateHash(@Nonnull final Vulnerability vulnerability) { CRC32 crc = new CRC32(); - crc.update(name().getBytes()); + update(crc, name()); final Evidence evidence = vulnerability.getEvidence(); if (evidence != null) { - crc.update(evidence.getValue().getBytes()); + update(crc, evidence.getValue()); } return crc.getValue(); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/AdapterFactory.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/AdapterFactory.java index 4c2d13fb57c..b62e74388ae 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/AdapterFactory.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/AdapterFactory.java @@ -37,7 +37,7 @@ static class Context { final List sources; final Map sourceIndexMap; final Map sourceContext; - Vulnerability vulnerability; + @Nullable Vulnerability vulnerability; public Context() { sources = new ArrayList<>(); @@ -83,6 +83,8 @@ public JsonAdapter create( return new EvidenceAdapter(moshi); } else if (VulnerabilityType.class.equals(rawType)) { return new VulnerabilityTypeAdapter(); + } else if (TruncatedVulnerabilities.class.equals(rawType)) { + return new TruncatedVulnerabilitiesAdapter(moshi); } return null; } @@ -181,7 +183,7 @@ public static class RedactionContext { private final Source source; private final boolean sensitive; private boolean sensitiveRanges; - private String redactedValue; + @Nullable private String redactedValue; public RedactionContext(final Source source) { this.source = source; @@ -204,6 +206,7 @@ public boolean shouldRedact() { return sensitive || sensitiveRanges; } + @Nullable public String getRedactedValue() { return redactedValue; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/EvidenceAdapter.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/EvidenceAdapter.java index 96adaec0f25..9c78f11ed81 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/EvidenceAdapter.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/EvidenceAdapter.java @@ -31,9 +31,13 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EvidenceAdapter extends FormattingAdapter { + private static final Logger log = LoggerFactory.getLogger(EvidenceAdapter.class); + private final JsonAdapter sourceAdapter; private final JsonAdapter defaultAdapter; private final JsonAdapter redactedAdapter; @@ -59,15 +63,23 @@ public void toJson(@Nonnull final JsonWriter writer, final @Nullable Evidence ev } private String substring(final String value, final Ranged range) { - final int end = Math.min(range.getStart() + range.getLength(), value.length()); + int end = Math.min(range.getStart() + range.getLength(), value.length()); + if (end < 0) { + log.debug("Invalid negative end parameter for substring. Value: {} Range: {}", value, range); + end = value.length(); + } return value.substring(range.getStart(), end); } private class DefaultEvidenceAdapter extends FormattingAdapter { @Override - public void toJson(@Nonnull final JsonWriter writer, final @Nonnull Evidence evidence) + public void toJson(@Nonnull final JsonWriter writer, final @Nullable Evidence evidence) throws IOException { + if (evidence == null) { + writer.nullValue(); + return; + } writer.beginObject(); if (evidence.getRanges() == null || evidence.getRanges().length == 0) { writer.name("value"); @@ -124,8 +136,12 @@ private void writeValuePart( private class RedactedEvidenceAdapter extends FormattingAdapter { @Override - public void toJson(@Nonnull final JsonWriter writer, @Nonnull final Evidence evidence) + public void toJson(@Nonnull final JsonWriter writer, @Nullable final Evidence evidence) throws IOException { + if (evidence == null) { + writer.nullValue(); + return; + } final Context ctx = Context.get(); final Vulnerability vulnerability = ctx.vulnerability; if (vulnerability == null) { @@ -155,7 +171,10 @@ private void toRedactedJson( writer.beginArray(); for (final Iterator it = new ValuePartIterator(ctx, value, tainted, sensitive); it.hasNext(); ) { - it.next().write(ctx, writer); + final ValuePart next = it.next(); + if (next != null) { + next.write(ctx, writer); + } } writer.endArray(); } @@ -197,6 +216,7 @@ public boolean hasNext() { return !next.isEmpty() || index < value.length(); } + @Nullable @Override public ValuePart next() { if (!hasNext()) { @@ -229,6 +249,7 @@ public ValuePart next() { return next.poll(); } + @Nullable private Ranged handleTaintedValue( @Nonnull final Range nextTainted, @Nullable Ranged nextSensitive) { final RedactionContext redactionCtx = ctx.getRedaction(nextTainted.getSource()); @@ -282,6 +303,7 @@ private void handleSensitiveValue(@Nonnull Ranged nextSensitive) { * Removes the tainted range from the sensitive one and returns whatever is before and enqueues * the rest */ + @Nullable private Ranged removeTaintedRange(final Ranged sensitive, final Range tainted) { final List disjointRanges = sensitive.remove(tainted); Ranged result = null; @@ -295,6 +317,7 @@ private Ranged removeTaintedRange(final Ranged sensitive, final Range tainted) { return result; } + @Nullable private ValuePart nextStringValuePart(final int end) { if (index < end) { final String chunk = value.substring(index, end); @@ -317,9 +340,10 @@ interface ValuePart { } static class StringValuePart implements ValuePart { - private final String value; - private StringValuePart(final String value) { + @Nullable private final String value; + + private StringValuePart(@Nullable final String value) { this.value = value; } @@ -435,11 +459,13 @@ private void addValuePart( valueParts.add(new TaintedValuePart(adapter, source, chunk, false)); } else { final int length = chunk.length(); - final int matching = source.getValue().indexOf(chunk); + final String sourceValue = source.getValue(); + final String redactedValue = ctx.getRedactedValue(); + final int matching = (sourceValue == null) ? -1 : sourceValue.indexOf(chunk); final String pattern; - if (matching >= 0) { + if (matching >= 0 && redactedValue != null) { // if matches append the matching part from the redacted value - pattern = ctx.getRedactedValue().substring(matching, matching + length); + pattern = redactedValue.substring(matching, matching + length); } else { // otherwise redact the string pattern = SensitiveHandler.get().redactString(chunk); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/SourceAdapter.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/SourceAdapter.java index 9da5177c6ef..7f5ab7edc1f 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/SourceAdapter.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/SourceAdapter.java @@ -66,7 +66,8 @@ public void toJson(@Nonnull final JsonWriter writer, final @Nonnull Source sourc } } - private void toRedactedJson(final JsonWriter writer, final Source source, final String value) + private void toRedactedJson( + final JsonWriter writer, final Source source, @Nullable final String value) throws IOException { writer.beginObject(); writer.name("origin"); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilities.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilities.java new file mode 100644 index 00000000000..fe767384ef2 --- /dev/null +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilities.java @@ -0,0 +1,19 @@ +package com.datadog.iast.model.json; + +import com.datadog.iast.model.Vulnerability; +import java.util.List; +import javax.annotation.Nullable; + +public class TruncatedVulnerabilities { + + @Nullable private final List vulnerabilities; + + public TruncatedVulnerabilities(@Nullable final List vulnerabilities) { + this.vulnerabilities = vulnerabilities; + } + + @Nullable + public List getVulnerabilities() { + return vulnerabilities; + } +} diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilitiesAdapter.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilitiesAdapter.java new file mode 100644 index 00000000000..ffe32b726bc --- /dev/null +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilitiesAdapter.java @@ -0,0 +1,87 @@ +package com.datadog.iast.model.json; + +import com.datadog.iast.model.Evidence; +import com.datadog.iast.model.Location; +import com.datadog.iast.model.Vulnerability; +import com.datadog.iast.model.VulnerabilityType; +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonWriter; +import com.squareup.moshi.Moshi; +import java.io.IOException; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +class TruncatedVulnerabilitiesAdapter extends FormattingAdapter { + + private static final String MAX_SIZE_EXCEEDED = "MAX SIZE EXCEEDED"; + + private final JsonAdapter vulnerabilityAdapter; + + public TruncatedVulnerabilitiesAdapter(Moshi moshi) { + this.vulnerabilityAdapter = new TruncatedVulnerabilityAdapter(moshi); + } + + @Override + public void toJson(@Nonnull JsonWriter writer, @Nullable TruncatedVulnerabilities value) + throws IOException { + if (value == null) { + writer.nullValue(); + return; + } + final List vulnerabilities = value.getVulnerabilities(); + writer.beginObject(); + if (vulnerabilities != null && !vulnerabilities.isEmpty()) { + writer.name("vulnerabilities"); + writer.beginArray(); + for (Vulnerability vulnerability : vulnerabilities) { + vulnerabilityAdapter.toJson(writer, vulnerability); + } + writer.endArray(); + } + writer.endObject(); + } + + private static class TruncatedVulnerabilityAdapter extends FormattingAdapter { + + private final JsonAdapter vulnerabilityTypeAdapter; + + private final JsonAdapter evidenceAdapter; + + private final JsonAdapter locationAdapter; + + public TruncatedVulnerabilityAdapter(Moshi moshi) { + this.vulnerabilityTypeAdapter = moshi.adapter(VulnerabilityType.class); + this.evidenceAdapter = new TruncatedEvidenceAdapter(); + this.locationAdapter = moshi.adapter(Location.class); + } + + @Override + public void toJson(@Nonnull JsonWriter writer, @Nullable Vulnerability value) + throws IOException { + if (value == null) { + return; + } + writer.beginObject(); + writer.name("type"); + vulnerabilityTypeAdapter.toJson(writer, value.getType()); + writer.name("evidence"); + evidenceAdapter.toJson(writer, value.getEvidence()); + writer.name("hash"); + writer.value(value.getHash()); + writer.name("location"); + locationAdapter.toJson(writer, value.getLocation()); + writer.endObject(); + } + } + + private static class TruncatedEvidenceAdapter extends FormattingAdapter { + @Override + public void toJson(@Nonnull JsonWriter writer, @Nullable Evidence evidence) throws IOException { + writer.beginObject(); + writer.name("value"); + writer.value(MAX_SIZE_EXCEEDED); + writer.endObject(); + } + } +} diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/VulnerabilityEncoding.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/VulnerabilityEncoding.java index e777953632a..59c6c1c5b62 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/VulnerabilityEncoding.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/VulnerabilityEncoding.java @@ -3,16 +3,37 @@ import com.datadog.iast.model.VulnerabilityBatch; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class VulnerabilityEncoding { + private static final Logger log = LoggerFactory.getLogger(VulnerabilityEncoding.class); + private static final int MAX_SPAN_TAG_SIZE = 25000; + static final Moshi MOSHI = new Moshi.Builder().add(new SourceTypeAdapter()).add(new AdapterFactory()).build(); private static final JsonAdapter BATCH_ADAPTER = MOSHI.adapter(VulnerabilityBatch.class); + private static final JsonAdapter TRUNCATED_VULNERABILITIES_ADAPTER = + MOSHI.adapter(TruncatedVulnerabilities.class); + public static String toJson(final VulnerabilityBatch value) { - return BATCH_ADAPTER.toJson(value); + try { + String json = BATCH_ADAPTER.toJson(value); + return json.getBytes().length > MAX_SPAN_TAG_SIZE + ? getExceededTagSizeJson(new TruncatedVulnerabilities(value.getVulnerabilities())) + : json; + } catch (Exception ex) { + log.debug("Vulnerability serialization error", ex); + return "{\"vulnerabilities\":[]}"; + } + } + + static String getExceededTagSizeJson(final TruncatedVulnerabilities truncatedVulnerabilities) { + // TODO report via telemetry + return TRUNCATED_VULNERABILITIES_ADAPTER.toJson(truncatedVulnerabilities); } } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operation.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operation.java index f0d75012887..c21cec9d9c7 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operation.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operation.java @@ -1,7 +1,9 @@ package com.datadog.iast.overhead; +import javax.annotation.Nullable; + public interface Operation { - boolean hasQuota(final OverheadContext context); + boolean hasQuota(@Nullable final OverheadContext context); - boolean consumeQuota(final OverheadContext context); + boolean consumeQuota(@Nullable final OverheadContext context); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operations.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operations.java index c9eab7b7237..e7bb35f1550 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operations.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/Operations.java @@ -1,5 +1,7 @@ package com.datadog.iast.overhead; +import javax.annotation.Nullable; + public class Operations { private Operations() {} @@ -7,7 +9,7 @@ private Operations() {} public static final Operation REPORT_VULNERABILITY = new Operation() { @Override - public boolean hasQuota(final OverheadContext context) { + public boolean hasQuota(@Nullable final OverheadContext context) { if (context == null) { return false; } @@ -15,7 +17,7 @@ public boolean hasQuota(final OverheadContext context) { } @Override - public boolean consumeQuota(final OverheadContext context) { + public boolean consumeQuota(@Nullable final OverheadContext context) { if (context == null) { return false; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/OverheadController.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/OverheadController.java index ee03973a11e..44f62063030 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/OverheadController.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/overhead/OverheadController.java @@ -13,6 +13,7 @@ import datadog.trace.util.AgentTaskScheduler; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,9 +25,9 @@ public interface OverheadController { int releaseRequest(); - boolean hasQuota(final Operation operation, final AgentSpan span); + boolean hasQuota(final Operation operation, @Nullable final AgentSpan span); - boolean consumeQuota(final Operation operation, final AgentSpan span); + boolean consumeQuota(final Operation operation, @Nullable final AgentSpan span); static OverheadController build(final Config config, final AgentTaskScheduler scheduler) { final OverheadControllerImpl result = new OverheadControllerImpl(config, scheduler); @@ -68,7 +69,7 @@ public int releaseRequest() { } @Override - public boolean hasQuota(final Operation operation, final AgentSpan span) { + public boolean hasQuota(final Operation operation, @Nullable final AgentSpan span) { final boolean result = delegate.hasQuota(operation, span); if (LOGGER.isDebugEnabled()) { LOGGER.debug( @@ -82,7 +83,7 @@ public boolean hasQuota(final Operation operation, final AgentSpan span) { } @Override - public boolean consumeQuota(final Operation operation, final AgentSpan span) { + public boolean consumeQuota(final Operation operation, @Nullable final AgentSpan span) { final boolean result = delegate.consumeQuota(operation, span); if (LOGGER.isDebugEnabled()) { LOGGER.debug( @@ -103,7 +104,7 @@ public void reset() { } } - private int getAvailableQuote(final AgentSpan span) { + private int getAvailableQuote(@Nullable final AgentSpan span) { final OverheadContext context = delegate.getContext(span); return context == null ? -1 : context.getAvailableQuota(); } @@ -149,16 +150,17 @@ public int releaseRequest() { } @Override - public boolean hasQuota(final Operation operation, final AgentSpan span) { + public boolean hasQuota(final Operation operation, @Nullable final AgentSpan span) { return operation.hasQuota(getContext(span)); } @Override - public boolean consumeQuota(final Operation operation, final AgentSpan span) { + public boolean consumeQuota(final Operation operation, @Nullable final AgentSpan span) { return operation.consumeQuota(getContext(span)); } - public OverheadContext getContext(final AgentSpan span) { + @Nullable + public OverheadContext getContext(@Nullable final AgentSpan span) { final RequestContext requestContext = span != null ? span.getRequestContext() : null; if (requestContext != null) { IastRequestContext iastRequestContext = requestContext.getData(RequestContextSlot.IAST); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/PropagationModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/PropagationModuleImpl.java index cae08236901..06d9e7cfeb8 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/PropagationModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/PropagationModuleImpl.java @@ -1,9 +1,8 @@ package com.datadog.iast.propagation; -import static com.datadog.iast.model.Range.NOT_MARKED; import static com.datadog.iast.taint.Ranges.highestPriorityRange; -import static com.datadog.iast.taint.Tainteds.canBeTainted; import static com.datadog.iast.util.ObjectVisitor.State.CONTINUE; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; @@ -11,346 +10,493 @@ import com.datadog.iast.taint.Ranges; import com.datadog.iast.taint.TaintedObject; import com.datadog.iast.taint.TaintedObjects; +import com.datadog.iast.taint.Tainteds; import com.datadog.iast.util.ObjectVisitor; -import com.datadog.iast.util.ObjectVisitor.State; -import com.datadog.iast.util.ObjectVisitor.Visitor; -import datadog.trace.api.iast.SourceTypes; +import datadog.trace.api.Config; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.Taintable; import datadog.trace.api.iast.propagation.PropagationModule; -import java.util.Collection; -import java.util.List; -import java.util.Map; import java.util.function.Predicate; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.jetbrains.annotations.Contract; public class PropagationModuleImpl implements PropagationModule { + /** Prevent copy of values bigger than this threshold */ + private static final int MAX_VALUE_LENGTH = Config.get().getIastTruncationMaxValueLength(); + @Override - public void taintIfInputIsTainted(@Nullable final Object toTaint, @Nullable final Object input) { - if (toTaint == null || input == null) { - return; - } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - final Source source = highestPriorityTaintedSource(taintedObjects, input); - if (source != null) { - taintObject(taintedObjects, toTaint, source); - } + public void taint(@Nullable final Object target, final byte origin) { + taint(target, origin, null); } @Override - public void taintIfInputIsTainted(@Nullable final String toTaint, @Nullable final Object input) { - if (!canBeTainted(toTaint) || input == null) { - return; - } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - final Source source = highestPriorityTaintedSource(taintedObjects, input); - if (source != null) { - taintString(taintedObjects, toTaint, source); - } + public void taint( + @Nullable final Object target, final byte origin, @Nullable final CharSequence name) { + taint(target, origin, name, sourceValue(target)); } @Override - public void taintIfInputIsTainted( + public void taint( + @Nullable final Object target, final byte origin, - @Nullable final String name, - @Nullable final String toTaint, - @Nullable final Object input) { - if (!canBeTainted(toTaint) || input == null) { + @Nullable final CharSequence name, + @Nullable final CharSequence value) { + if (!canBeTainted(target)) { return; } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - if (isTainted(taintedObjects, input)) { - taintString(taintedObjects, toTaint, new Source(origin, name, toTaint)); - } + taint(LazyContext.build(), target, origin, name, value); } @Override - public void taintIfInputIsTainted( - final byte origin, - @Nullable final String name, - @Nullable final Collection toTaintCollection, - @Nullable final Object input) { - if (toTaintCollection == null || toTaintCollection.isEmpty() || input == null) { - return; - } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - if (isTainted(taintedObjects, input)) { - for (final String toTaint : toTaintCollection) { - if (canBeTainted(toTaint)) { - taintString(taintedObjects, toTaint, new Source(origin, name, toTaint)); - } - } - } + public void taint( + @Nullable final IastContext ctx, @Nullable final Object target, final byte origin) { + taint(ctx, target, origin, null); } @Override - public void taintIfInputIsTainted( + public void taint( + @Nullable final IastContext ctx, + @Nullable final Object target, final byte origin, - @Nullable final Collection toTaintCollection, - @Nullable final Object input) { - if (toTaintCollection == null || toTaintCollection.isEmpty() || input == null) { - return; - } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - if (isTainted(taintedObjects, input)) { - for (final String toTaint : toTaintCollection) { - if (canBeTainted(toTaint)) { - taintString(taintedObjects, toTaint, new Source(origin, toTaint, toTaint)); - } - } - } + @Nullable final CharSequence name) { + taint(ctx, target, origin, name, sourceValue(target)); } @Override - public void taintIfInputIsTainted( + public void taint( + @Nullable final IastContext ctx, + @Nullable final Object target, final byte origin, - @Nullable final List> toTaintCollection, - @Nullable final Object input) { - if (toTaintCollection == null || toTaintCollection.isEmpty() || input == null) { + @Nullable final CharSequence name, + @Nullable final CharSequence value) { + if (!canBeTainted(target)) { return; } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - if (isTainted(taintedObjects, input)) { - for (final Map.Entry entry : toTaintCollection) { - final String name = entry.getKey(); - if (canBeTainted(name)) { - taintString( - taintedObjects, name, new Source(SourceTypes.namedSource(origin), name, name)); - } - final String toTaint = entry.getValue(); - if (canBeTainted(toTaint)) { - taintString(taintedObjects, toTaint, new Source(origin, name, toTaint)); - } - } - } + internalTaint(ctx, target, newSource(target, origin, name, value), NOT_MARKED); } @Override - public void taintIfAnyInputIsTainted( - @Nullable final Object toTaint, @Nullable final Object... inputs) { - if (toTaint == null || inputs == null || inputs.length == 0) { + public void taintIfTainted(@Nullable final Object target, @Nullable final Object input) { + taintIfTainted(target, input, false, NOT_MARKED); + } + + @Override + public void taintIfTainted( + @Nullable final Object target, @Nullable final Object input, boolean keepRanges, int mark) { + if (!canBeTainted(target) || !canBeTainted(input)) { return; } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - for (final Object input : inputs) { - final Source source = highestPriorityTaintedSource(taintedObjects, input); - if (source != null) { - taintObject(taintedObjects, toTaint, source); - return; - } - } + taintIfTainted(LazyContext.build(), target, input, keepRanges, mark); } @Override - public void taint(final byte source, @Nullable final String name, @Nullable final String value) { - if (!canBeTainted(value)) { + public void taintIfTainted( + @Nullable final IastContext ctx, + @Nullable final Object target, + @Nullable final Object input) { + taintIfTainted(ctx, target, input, false, NOT_MARKED); + } + + @Override + public void taintIfTainted( + @Nullable final IastContext ctx, + @Nullable final Object target, + @Nullable final Object input, + boolean keepRanges, + int mark) { + if (!canBeTainted(target) || !canBeTainted(input)) { return; } - final IastRequestContext ctx = IastRequestContext.get(); - if (ctx == null) { - return; + if (keepRanges) { + internalTaint(ctx, target, getRanges(ctx, input), mark); + } else { + internalTaint(ctx, target, highestPrioritySource(ctx, input), mark); } - final TaintedObjects taintedObjects = ctx.getTaintedObjects(); - taintedObjects.taintInputString(value, new Source(source, name, value)); } @Override - public void taint( - @Nullable final Object ctx_, - final byte source, - @Nullable final String name, - @Nullable final String value) { - if (ctx_ == null || !canBeTainted(value)) { - return; - } - final IastRequestContext ctx = (IastRequestContext) ctx_; - final TaintedObjects taintedObjects = ctx.getTaintedObjects(); - taintedObjects.taintInputString(value, new Source(source, name, value)); + public void taintIfTainted( + @Nullable final Object target, @Nullable final Object input, final byte origin) { + taintIfTainted(target, input, origin, null); } @Override - public void taintDeeply(@Nullable final Object ctx_, final byte source, @Nonnull final Object o) { - taintDeeply(ctx_, source, o, ObjectVisitor::inspectClass); + public void taintIfTainted( + @Nullable final Object target, + @Nullable final Object input, + final byte origin, + @Nullable final CharSequence name) { + taintIfTainted(target, input, origin, name, sourceValue(target)); } @Override - public void taintDeeply( - @Nullable final Object ctx_, - final byte source, - @Nonnull final Object o, - @Nonnull final Predicate> classFilter) { - if (ctx_ == null) { + public void taintIfTainted( + @Nullable final Object target, + @Nullable final Object input, + final byte origin, + @Nullable final CharSequence name, + @Nullable final CharSequence value) { + if (!canBeTainted(target) || !canBeTainted(input)) { return; } - final IastRequestContext ctx = (IastRequestContext) ctx_; - final TaintedObjects taintedObjects = ctx.getTaintedObjects(); - ObjectVisitor.visit(o, new TaintingVisitor(taintedObjects, source), classFilter); + taintIfTainted(LazyContext.build(), target, input, origin, name, value); } @Override - public void taintObjectIfInputIsTaintedKeepingRanges( - @Nullable final Object toTaint, @Nullable Object input) { - if (toTaint == null || input == null) { - return; - } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - final Range[] ranges = getTaintedRanges(taintedObjects, input); - if (ranges != null && ranges.length > 0) { - taintedObjects.taint(toTaint, ranges); - } + public void taintIfTainted( + @Nullable final IastContext ctx, + @Nullable final Object target, + @Nullable final Object input, + final byte origin) { + taintIfTainted(ctx, target, input, origin, null); + } + + @Override + public void taintIfTainted( + @Nullable final IastContext ctx, + @Nullable final Object target, + @Nullable final Object input, + final byte origin, + @Nullable final CharSequence name) { + taintIfTainted(ctx, target, input, origin, name, sourceValue(target)); } @Override - public void taintObject(final byte origin, @Nullable final Object toTaint) { - if (toTaint == null) { + public void taintIfTainted( + @Nullable final IastContext ctx, + @Nullable final Object target, + @Nullable final Object input, + final byte origin, + @Nullable final CharSequence name, + @Nullable final CharSequence value) { + if (!canBeTainted(target) || !canBeTainted(input)) { return; } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(false); - if (taintedObjects == null) { - return; + if (isTainted(ctx, input)) { + internalTaint(ctx, target, newSource(target, origin, name, value), NOT_MARKED); } - final Source source = new Source(origin, null, null); - taintObject(taintedObjects, toTaint, source); } @Override - public void taintObjects(final byte origin, @Nullable final Object[] toTaintArray) { - if (toTaintArray == null || toTaintArray.length == 0) { + public void taintIfAnyTainted(@Nullable final Object target, @Nullable final Object[] inputs) { + taintIfAnyTainted(target, inputs, false, NOT_MARKED); + } + + @Override + public void taintIfAnyTainted( + @Nullable final Object target, + @Nullable final Object[] inputs, + final boolean keepRanges, + final int mark) { + if (!canBeTainted(target) || !canBeTainted(inputs)) { return; } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - final Source source = new Source(origin, null, null); - for (final Object toTaint : toTaintArray) { - taintObject(taintedObjects, toTaint, source); - } + taintIfAnyTainted(LazyContext.build(), target, inputs, keepRanges, mark); } @Override - public boolean isTainted(@Nullable Object obj) { - if (obj instanceof Taintable) { - return ((Taintable) obj).$DD$isTainted(); - } + public void taintIfAnyTainted( + @Nullable final IastContext ctx, + @Nullable final Object target, + @Nullable final Object[] inputs) { + taintIfAnyTainted(ctx, target, inputs, false, NOT_MARKED); + } - if (obj == null) { - return false; + @Override + public void taintIfAnyTainted( + @Nullable final IastContext ctx, + @Nullable final Object target, + @Nullable final Object[] inputs, + final boolean keepRanges, + final int mark) { + if (!canBeTainted(target) || !canBeTainted(inputs)) { + return; + } + if (keepRanges) { + final Range[] ranges = getRangesInArray(ctx, inputs); + if (ranges != null) { + internalTaint(ctx, target, ranges, mark); + } + } else { + final Source source = highestPrioritySourceInArray(ctx, inputs); + if (source != null) { + internalTaint(ctx, target, source, mark); + } } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - return taintedObjects.get(obj) != null; } @Override - public void taintObjects( - final byte origin, @Nullable final Collection toTaintCollection) { - if (toTaintCollection == null || toTaintCollection.isEmpty()) { + public void taintDeeply( + @Nullable final Object target, final byte origin, final Predicate> classFilter) { + if (!canBeTainted(target)) { return; } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - final Source source = new Source(origin, null, null); - for (final Object toTaint : toTaintCollection) { - taintObject(taintedObjects, toTaint, source); - } + taintDeeply(LazyContext.build(), target, origin, classFilter); } @Override - public void taintObject( - byte origin, @Nullable String name, @Nullable String value, @Nullable Object t) { - if (t == null) { + public void taintDeeply( + @Nullable final IastContext ctx, + @Nullable final Object target, + final byte origin, + final Predicate> classFilter) { + if (!canBeTainted(target)) { return; } - if (t instanceof Taintable) { - ((Taintable) t).$$DD$setSource(new Source(origin, name, value)); + final TaintedObjects to = getTaintedObjects(ctx); + if (to == null) { + return; + } + if (target instanceof CharSequence) { + internalTaint(ctx, target, newSource(target, origin, null, sourceValue(target)), NOT_MARKED); } else { - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(); - taintObject(taintedObjects, t, new Source(origin, name, value)); + ObjectVisitor.visit(target, new TaintingVisitor(to, origin), classFilter); } } + @Nullable + @Override + public Taintable.Source findSource(@Nullable final Object target) { + return target == null ? null : findSource(LazyContext.build(), target); + } + + @Nullable @Override - public Taintable.Source firstTaintedSource(@Nullable final Object input) { - if (input == null) { + public Taintable.Source findSource( + @Nullable final IastContext ctx, @Nullable final Object target) { + if (target == null) { return null; } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - return highestPriorityTaintedSource(taintedObjects, input); + return highestPrioritySource(ctx, target); } @Override - public void taintIfInputIsTaintedWithMarks( - @Nullable final String toTaint, @Nullable final Object input, final int mark) { - if (!canBeTainted(toTaint) || input == null) { - return; + public boolean isTainted(@Nullable final Object target) { + return target != null && isTainted(LazyContext.build(), target); + } + + @Override + public boolean isTainted(@Nullable final IastContext ctx, @Nullable final Object target) { + return target != null && findSource(ctx, target) != null; + } + + /** Ensures that the reference is not kept strongly via the name or value properties */ + private static Source newSource( + @Nonnull final Object reference, + final byte origin, + @Nullable final CharSequence name, + @Nullable final CharSequence value) { + return new Source( + origin, + reference == name ? sourceValue(name) : name, + reference == value ? sourceValue(value) : value); + } + + /** + * This method will prevent the code from creating a strong reference to what should remain weak + */ + @Nullable + private static CharSequence sourceValue(@Nullable final Object target) { + if (target instanceof String) { + final String string = (String) target; + if (MAX_VALUE_LENGTH > string.length()) { + return String.copyValueOf(string.toCharArray()); + } else { + final char[] chars = new char[MAX_VALUE_LENGTH]; + string.getChars(0, MAX_VALUE_LENGTH, chars, 0); + return String.copyValueOf(chars); + } + } else if (target instanceof CharSequence) { + final CharSequence charSequence = (CharSequence) target; + if (MAX_VALUE_LENGTH > charSequence.length()) { + return charSequence.toString(); + } else { + final CharSequence subSequence = charSequence.subSequence(0, MAX_VALUE_LENGTH); + return subSequence.toString(); + } } - final TaintedObjects taintedObjects = TaintedObjects.activeTaintedObjects(true); - final Range[] ranges = getTaintedRanges(taintedObjects, input); - if (ranges != null && ranges.length > 0) { - Range priorityRange = highestPriorityRange(ranges); - taintedObjects.taintInputString( - toTaint, priorityRange.getSource(), priorityRange.getMarks() | mark); + return null; + } + + @Contract("null -> false") + private static boolean canBeTainted(@Nullable final Object target) { + if (target == null) { + return false; } + if (target instanceof CharSequence) { + return Tainteds.canBeTainted((CharSequence) target); + } + return true; } - private static void taintString( - final TaintedObjects taintedObjects, final String toTaint, final Source source) { - taintedObjects.taintInputString(toTaint, source); + @Contract("null -> false") + private static boolean canBeTainted(@Nullable final Object[] target) { + if (target == null || target.length == 0) { + return false; + } + return true; } - private static void taintObject( - final TaintedObjects taintedObjects, final Object toTaint, final Source source) { - if (toTaint instanceof Taintable) { - ((Taintable) toTaint).$$DD$setSource(source); - } else { - taintedObjects.taintInputObject(toTaint, source); + @Nullable + private static TaintedObjects getTaintedObjects(final @Nullable IastContext ctx) { + IastRequestContext iastCtx = null; + if (ctx instanceof IastRequestContext) { + iastCtx = (IastRequestContext) ctx; + } else if (ctx instanceof LazyContext) { + iastCtx = ((LazyContext) ctx).getDelegate(); } + return iastCtx == null ? null : iastCtx.getTaintedObjects(); } - private static boolean isTainted(final TaintedObjects taintedObjects, final Object object) { - return highestPriorityTaintedSource(taintedObjects, object) != null; + @Nullable + private static Range[] getRangesInArray( + final @Nullable IastContext ctx, final @Nonnull Object[] objects) { + for (final Object object : objects) { + final Range[] ranges = getRanges(ctx, object); + if (ranges != null) { + return ranges; + } + } + return null; } - private static Source highestPriorityTaintedSource( - final TaintedObjects taintedObjects, final Object object) { + @Nullable + private static Range[] getRanges(final @Nullable IastContext ctx, final @Nonnull Object object) { + if (object instanceof Taintable) { + final Source source = highestPrioritySource(ctx, object); + if (source == null) { + return null; + } else { + return new Range[] {new Range(0, Integer.MAX_VALUE, source, NOT_MARKED)}; + } + } + final TaintedObjects to = getTaintedObjects(ctx); + if (to == null) { + return null; + } + final TaintedObject tainted = to.get(object); + return tainted == null ? null : tainted.getRanges(); + } + + @Nullable + private static Source highestPrioritySourceInArray( + final @Nullable IastContext ctx, final @Nonnull Object[] objects) { + for (final Object object : objects) { + final Source source = highestPrioritySource(ctx, object); + if (source != null) { + return source; + } + } + return null; + } + + @Nullable + private static Source highestPrioritySource( + final @Nullable IastContext ctx, final @Nonnull Object object) { if (object instanceof Taintable) { return (Source) ((Taintable) object).$$DD$getSource(); } else { - final TaintedObject tainted = taintedObjects.get(object); - final Range[] ranges = tainted == null ? null : tainted.getRanges(); + final Range[] ranges = getRanges(ctx, object); return ranges != null && ranges.length > 0 ? highestPriorityRange(ranges).getSource() : null; } } - private static Range[] getTaintedRanges( - final TaintedObjects taintedObjects, final Object object) { - if (object instanceof Taintable) { - Source source = (Source) ((Taintable) object).$$DD$getSource(); - if (source == null) { - return null; + private static void internalTaint( + @Nullable final IastContext ctx, + @Nonnull final Object value, + @Nullable final Source source, + int mark) { + if (source == null) { + return; + } + if (value instanceof Taintable) { + ((Taintable) value).$$DD$setSource(source); + } else { + final TaintedObjects to = getTaintedObjects(ctx); + if (to == null) { + return; + } + if (value instanceof CharSequence) { + to.taint(value, Ranges.forCharSequence((CharSequence) value, source, mark)); } else { - return Ranges.forObject(source, NOT_MARKED); + to.taint(value, Ranges.forObject(source, mark)); } + } + } + + private static void internalTaint( + @Nullable final IastContext ctx, + @Nonnull final Object value, + @Nullable final Range[] ranges, + final int mark) { + if (ranges == null || ranges.length == 0) { + return; + } + if (value instanceof Taintable) { + ((Taintable) value).$$DD$setSource(ranges[0].getSource()); } else { - final TaintedObject tainted = taintedObjects.get(object); - return tainted == null ? null : tainted.getRanges(); + final TaintedObjects to = getTaintedObjects(ctx); + if (to != null) { + final Range[] markedRanges = markRanges(ranges, mark); + to.taint(value, markedRanges); + } + } + } + + @Nonnull + private static Range[] markRanges(@Nonnull final Range[] ranges, final int mark) { + if (mark == NOT_MARKED) { + return ranges; + } + final Range[] result = new Range[ranges.length]; + for (int i = 0; i < ranges.length; i++) { + final Range range = ranges[i]; + final int newMark = range.getMarks() | mark; + result[i] = new Range(range.getStart(), range.getLength(), range.getSource(), newMark); + } + return result; + } + + private static class LazyContext implements IastContext { + + private boolean fetched; + @Nullable private IastRequestContext delegate; + + @Nullable + private IastRequestContext getDelegate() { + if (!fetched) { + fetched = true; + delegate = IastRequestContext.get(); + } + return delegate; + } + + public static IastContext build() { + return new LazyContext(); } } - private static class TaintingVisitor implements Visitor { + private static class TaintingVisitor implements ObjectVisitor.Visitor { private final TaintedObjects taintedObjects; - private final byte source; + private final byte origin; - private TaintingVisitor(final TaintedObjects taintedObjects, final byte source) { + private TaintingVisitor(@Nonnull final TaintedObjects taintedObjects, final byte origin) { this.taintedObjects = taintedObjects; - this.source = source; + this.origin = origin; } @Nonnull @Override - public State visit(@Nonnull final String path, @Nonnull final Object value) { - if (value instanceof String) { - final String stringValue = (String) value; - if (canBeTainted(stringValue)) { - taintedObjects.taintInputString(stringValue, new Source(source, path, stringValue)); + public ObjectVisitor.State visit(@Nonnull final String path, @Nonnull final Object value) { + if (value instanceof CharSequence) { + final CharSequence charSequence = (CharSequence) value; + if (canBeTainted(charSequence)) { + final Source source = newSource(value, origin, path, charSequence); + taintedObjects.taint( + charSequence, Ranges.forCharSequence(charSequence, source, NOT_MARKED)); } } return CONTINUE; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java index 7a3f7720112..a93d3c53e8e 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java @@ -39,6 +39,7 @@ public class StringModuleImpl implements StringModule { private static final int NULL_STR_LENGTH = "null".length(); + @SuppressWarnings("NullAway") // NullAway fails with taintedLeft and taintedRight checks @Override public void onStringConcat( @Nonnull final String left, @Nullable final String right, @Nonnull final String result) { @@ -170,7 +171,7 @@ public void onStringConcatFactory( int offset = 0, rangeIndex = 0; for (int item : recipeOffsets) { if (item < 0) { - offset += (-item); + offset += -item; } else { final String argument = args[item]; final Range[] ranges = sourceRanges.get(item); @@ -214,7 +215,7 @@ public void onStringSubSequence( @Override public void onStringJoin( - @Nullable String result, @Nonnull CharSequence delimiter, @Nonnull CharSequence... elements) { + @Nullable String result, @Nonnull CharSequence delimiter, @Nonnull CharSequence[] elements) { if (!canBeTainted(result)) { return; } @@ -262,7 +263,7 @@ public void onStringJoin( @Override @SuppressFBWarnings("ES_COMPARING_PARAMETER_STRING_WITH_EQ") - public void onStringRepeat(String self, int count, String result) { + public void onStringRepeat(@Nonnull String self, int count, @Nonnull String result) { if (!canBeTainted(self) || !canBeTainted(result) || self == result) { return; } @@ -368,7 +369,7 @@ private static int insertRange( return rangeIndex + count; } - private static Range[] getRanges(final TaintedObject taintedObject) { + private static Range[] getRanges(@Nullable final TaintedObject taintedObject) { return taintedObject == null ? EMPTY : taintedObject.getRanges(); } @@ -572,11 +573,11 @@ public void onSplit(@Nonnull String self, @Nonnull String[] result) { * @param finalRanges result with all ranges */ private void addParameterTaintedRanges( - final Range placeholderRange, + @Nullable final Range placeholderRange, final Object param, final String formatted, final int offset, - final Range[] ranges, + @Nullable final Range[] ranges, /* out */ final RangeList finalRanges) { if (ranges != null && ranges.length > 0) { // only shift ranges if they are character sequences of the same length, otherwise taint the @@ -602,6 +603,7 @@ private void addParameterTaintedRanges( * @param finalRanges result with all ranges * @return tainted range of the placeholder or {@code null} if not tainted */ + @Nullable private Range addFormatTaintedRanges( final Ranged placeholderPos, final int offset, diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/AbstractRegexTokenizer.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/AbstractRegexTokenizer.java index 58a1e8aaec3..54c73669be3 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/AbstractRegexTokenizer.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/AbstractRegexTokenizer.java @@ -4,11 +4,12 @@ import java.util.NoSuchElementException; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nullable; public abstract class AbstractRegexTokenizer implements SensitiveHandler.Tokenizer { protected final Matcher matcher; - private Ranged current; + @Nullable private Ranged current; protected AbstractRegexTokenizer(final Pattern pattern, final String evidence) { matcher = pattern.matcher(evidence); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SensitiveHandlerImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SensitiveHandlerImpl.java index ffa8dbe3346..d62fdf8c95c 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SensitiveHandlerImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SensitiveHandlerImpl.java @@ -76,7 +76,7 @@ public Tokenizer tokenizeEvidence( return supplier.tokenizerFor(evidence); } - private int computeLength(final String value) { + private int computeLength(@Nullable final String value) { if (value == null || value.isEmpty()) { return 0; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SqlRegexpTokenizer.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SqlRegexpTokenizer.java index 9082979e890..6c87aaf7a87 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SqlRegexpTokenizer.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/SqlRegexpTokenizer.java @@ -120,10 +120,6 @@ public static Dialect fromEvidence(final Evidence evidence) { return ANSI; } - public static Dialect current() { - return ANSI; - } - public Pattern buildPattern() { return pattern.get(); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/TaintedRangeBasedTokenizer.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/TaintedRangeBasedTokenizer.java index b0f08bb9afb..b2cff3356b4 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/TaintedRangeBasedTokenizer.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sensitive/TaintedRangeBasedTokenizer.java @@ -2,22 +2,24 @@ import com.datadog.iast.model.Evidence; import com.datadog.iast.model.Range; +import com.datadog.iast.taint.Ranges; import com.datadog.iast.util.Ranged; import java.util.NoSuchElementException; +import javax.annotation.Nullable; public class TaintedRangeBasedTokenizer implements SensitiveHandler.Tokenizer { private final String value; private final Range[] ranges; - private Ranged current; + @Nullable private Ranged current; private int rangesIndex; private int pos; public TaintedRangeBasedTokenizer(final Evidence evidence) { - this.ranges = evidence.getRanges(); + this.ranges = evidence.getRanges() == null ? Ranges.EMPTY : evidence.getRanges(); this.value = evidence.getValue(); rangesIndex = 0; pos = 0; // current value position @@ -37,6 +39,7 @@ public Ranged current() { return current; } + @Nullable private Ranged buildNext() { for (; rangesIndex < ranges.length; rangesIndex++) { Range range = ranges[rangesIndex]; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/CommandInjectionModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/CommandInjectionModuleImpl.java index ea5fa289e6d..24fd88c99c4 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/CommandInjectionModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/CommandInjectionModuleImpl.java @@ -3,6 +3,7 @@ import static com.datadog.iast.taint.Ranges.rangesProviderFor; import static com.datadog.iast.taint.Tainteds.canBeTainted; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.VulnerabilityType; import com.datadog.iast.taint.TaintedObjects; @@ -15,6 +16,10 @@ public class CommandInjectionModuleImpl extends SinkModuleBase implements CommandInjectionModule { + public CommandInjectionModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onRuntimeExec(@Nullable final String... cmdArray) { if (!canBeTainted(cmdArray)) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HstsMissingHeaderModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HstsMissingHeaderModuleImpl.java index f096f5f46ce..801d7e51052 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HstsMissingHeaderModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HstsMissingHeaderModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Location; import com.datadog.iast.model.Vulnerability; @@ -23,6 +24,10 @@ public class HstsMissingHeaderModuleImpl extends SinkModuleBase implements HstsM private static final Logger LOGGER = LoggerFactory.getLogger(HstsMissingHeaderModuleImpl.class); + public HstsMissingHeaderModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onRequestEnd(final Object iastRequestContextObject, final IGSpanInfo igSpanInfo) { @@ -70,7 +75,7 @@ static boolean isHttps(@Nullable final String urlString, @Nullable final String if (urlString == null) { return false; } - if (urlString.toLowerCase().startsWith("https://")) { + if (urlString.toLowerCase(Locale.ROOT).startsWith("https://")) { return true; } if (forwardedFor == null) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HttpResponseHeaderModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HttpResponseHeaderModuleImpl.java index 3216bff3d60..9eea7be16c5 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HttpResponseHeaderModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/HttpResponseHeaderModuleImpl.java @@ -3,6 +3,7 @@ import static com.datadog.iast.util.HttpHeader.Values.SET_COOKIE; import static java.util.Collections.singletonList; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Evidence; import com.datadog.iast.model.Location; @@ -27,6 +28,10 @@ public class HttpResponseHeaderModuleImpl extends SinkModuleBase implements HttpResponseHeaderModule { + public HttpResponseHeaderModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onHeader(@Nonnull final String name, final String value) { final HttpHeader header = HttpHeader.from(name); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/LdapInjectionModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/LdapInjectionModuleImpl.java index 1b0fc2a3cd9..aaa0f999bfb 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/LdapInjectionModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/LdapInjectionModuleImpl.java @@ -3,6 +3,7 @@ import static com.datadog.iast.taint.Ranges.rangesProviderFor; import static com.datadog.iast.taint.Tainteds.canBeTainted; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.VulnerabilityType; import com.datadog.iast.taint.TaintedObjects; @@ -14,6 +15,10 @@ public class LdapInjectionModuleImpl extends SinkModuleBase implements LdapInjectionModule { + public LdapInjectionModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @SuppressWarnings("unchecked") @Override public void onDirContextSearch( diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/PathTraversalModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/PathTraversalModuleImpl.java index 2cb6eb4d816..cd5a54862a7 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/PathTraversalModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/PathTraversalModuleImpl.java @@ -4,6 +4,7 @@ import static com.datadog.iast.taint.Tainteds.canBeTainted; import static java.util.Arrays.asList; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.VulnerabilityType; import com.datadog.iast.taint.TaintedObjects; @@ -20,6 +21,10 @@ public class PathTraversalModuleImpl extends SinkModuleBase implements PathTraversalModule { + public PathTraversalModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onPathTraversal(final @Nullable String path) { if (!canBeTainted(path)) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SinkModuleBase.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SinkModuleBase.java index 57a7de1b0dc..d1a24a5516f 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SinkModuleBase.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SinkModuleBase.java @@ -3,7 +3,7 @@ import static com.datadog.iast.util.ObjectVisitor.State.CONTINUE; import static com.datadog.iast.util.ObjectVisitor.State.EXIT; -import com.datadog.iast.HasDependencies; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.Reporter; import com.datadog.iast.model.Evidence; @@ -29,14 +29,13 @@ /** Base class with utility methods for with sinks */ @SuppressWarnings({"UnusedReturnValue", "SameParameterValue"}) -public abstract class SinkModuleBase implements HasDependencies { +public abstract class SinkModuleBase { - protected OverheadController overheadController; - protected Reporter reporter; - protected StackWalker stackWalker; + protected final OverheadController overheadController; + protected final Reporter reporter; + protected final StackWalker stackWalker; - @Override - public void registerDependencies(@Nonnull final Dependencies dependencies) { + protected SinkModuleBase(@Nonnull final Dependencies dependencies) { overheadController = dependencies.getOverheadController(); reporter = dependencies.getReporter(); stackWalker = dependencies.getStackWalker(); @@ -89,6 +88,9 @@ public void registerDependencies(@Nonnull final Dependencies dependencies) { if (rangeProvider.size() == 1) { // only one item and has ranges final E item = rangeProvider.value(0); + if (item == null) { + return null; // should never happen + } evidence = item.toString(); targetRanges = rangeProvider.ranges(item); } else { @@ -177,10 +179,6 @@ protected StackTraceElement getCurrentStackTrace() { return stackWalker.walk(SinkModuleBase::findValidPackageForVulnerability); } - protected String getServiceName(final AgentSpan span) { - return span != null ? span.getServiceName() : null; - } - static StackTraceElement findValidPackageForVulnerability( @Nonnull final Stream stream) { final StackTraceElement[] first = new StackTraceElement[1]; @@ -198,13 +196,13 @@ static StackTraceElement findValidPackageForVulnerability( private class InjectionVisitor implements Visitor { - private final AgentSpan span; + @Nullable private final AgentSpan span; private final IastRequestContext ctx; private final InjectionType type; - private Evidence evidence; + @Nullable private Evidence evidence; private InjectionVisitor( - final AgentSpan span, final IastRequestContext ctx, final InjectionType type) { + @Nullable final AgentSpan span, final IastRequestContext ctx, final InjectionType type) { this.span = span; this.ctx = ctx; this.type = type; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SqlInjectionModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SqlInjectionModuleImpl.java index 31d42394329..98ad3d757f4 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SqlInjectionModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SqlInjectionModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Evidence; import com.datadog.iast.model.VulnerabilityType; @@ -10,6 +11,10 @@ public class SqlInjectionModuleImpl extends SinkModuleBase implements SqlInjectionModule { + public SqlInjectionModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onJdbcQuery(@Nullable final String queryString) { onJdbcQuery(queryString, null); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SsrfModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SsrfModuleImpl.java index b4df8302c94..e397e649c6e 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SsrfModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/SsrfModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.VulnerabilityType; import datadog.trace.api.iast.sink.SsrfModule; @@ -9,6 +10,10 @@ public class SsrfModuleImpl extends SinkModuleBase implements SsrfModule { + public SsrfModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onURLConnection(@Nullable final Object url) { if (url == null) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/StacktraceLeakModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/StacktraceLeakModuleImpl.java new file mode 100644 index 00000000000..2cfb07e5f49 --- /dev/null +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/StacktraceLeakModuleImpl.java @@ -0,0 +1,37 @@ +package com.datadog.iast.sink; + +import com.datadog.iast.Dependencies; +import com.datadog.iast.model.Evidence; +import com.datadog.iast.model.Location; +import com.datadog.iast.model.Vulnerability; +import com.datadog.iast.model.VulnerabilityType; +import datadog.trace.api.iast.sink.StacktraceLeakModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import org.jetbrains.annotations.NotNull; + +public class StacktraceLeakModuleImpl extends SinkModuleBase implements StacktraceLeakModule { + + public StacktraceLeakModuleImpl(@NotNull Dependencies dependencies) { + super(dependencies); + } + + @Override + public void onStacktraceLeak( + Throwable throwable, String moduleName, String className, String methodName) { + if (throwable != null) { + final AgentSpan span = AgentTracer.activeSpan(); + + Evidence evidence = + new Evidence( + "ExceptionHandler in " + + moduleName + + " \r\nthrown " + + throwable.getClass().getName()); + Location location = Location.forSpanAndClassAndMethod(span, className, methodName); + + reporter.report( + span, new Vulnerability(VulnerabilityType.STACKTRACE_LEAK, location, evidence)); + } + } +} diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/TrustBoundaryViolationModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/TrustBoundaryViolationModuleImpl.java index a584d078a0c..5f3ed0707b2 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/TrustBoundaryViolationModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/TrustBoundaryViolationModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.VulnerabilityType; import datadog.trace.api.iast.sink.TrustBoundaryViolationModule; @@ -9,6 +10,11 @@ public class TrustBoundaryViolationModuleImpl extends SinkModuleBase implements TrustBoundaryViolationModule { + + public TrustBoundaryViolationModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onSessionValue(@Nonnull String name, Object value) { final AgentSpan span = AgentTracer.activeSpan(); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/UnvalidatedRedirectModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/UnvalidatedRedirectModuleImpl.java index 45bd6ed2e47..deaf1977898 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/UnvalidatedRedirectModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/UnvalidatedRedirectModuleImpl.java @@ -2,6 +2,7 @@ import static com.datadog.iast.taint.Tainteds.canBeTainted; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Evidence; import com.datadog.iast.model.Location; @@ -25,6 +26,10 @@ public class UnvalidatedRedirectModuleImpl extends SinkModuleBase private static final String LOCATION_HEADER = "Location"; private static final String REFERER = "Referer"; + public UnvalidatedRedirectModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onRedirect(final @Nullable String value) { if (!canBeTainted(value)) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakCipherModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakCipherModuleImpl.java index f06a52c4f02..0c61867614d 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakCipherModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakCipherModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.model.Evidence; import com.datadog.iast.model.VulnerabilityType; import com.datadog.iast.overhead.Operations; @@ -14,9 +15,8 @@ public class WeakCipherModuleImpl extends SinkModuleBase implements WeakCipherMo private Config config; - @Override - public void registerDependencies(@Nonnull Dependencies dependencies) { - super.registerDependencies(dependencies); + public WeakCipherModuleImpl(final Dependencies dependencies) { + super(dependencies); config = dependencies.getConfig(); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakHashModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakHashModuleImpl.java index 3d91d1d371b..88b9cb8b0a4 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakHashModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakHashModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.model.Evidence; import com.datadog.iast.model.VulnerabilityType; import com.datadog.iast.overhead.Operations; @@ -14,9 +15,8 @@ public class WeakHashModuleImpl extends SinkModuleBase implements WeakHashModule private Config config; - @Override - public void registerDependencies(@Nonnull Dependencies dependencies) { - super.registerDependencies(dependencies); + public WeakHashModuleImpl(final Dependencies dependencies) { + super(dependencies); config = dependencies.getConfig(); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakRandomnessModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakRandomnessModuleImpl.java index d374927173a..7c13313eb30 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakRandomnessModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/WeakRandomnessModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.model.Evidence; import com.datadog.iast.model.VulnerabilityType; import com.datadog.iast.overhead.Operations; @@ -11,6 +12,10 @@ public class WeakRandomnessModuleImpl extends SinkModuleBase implements WeakRandomnessModule { + public WeakRandomnessModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onWeakRandom(@Nonnull final Class instance) { if (isSecuredInstance(instance)) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XContentTypeModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XContentTypeModuleImpl.java index 3d610e75792..8cb864e443e 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XContentTypeModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XContentTypeModuleImpl.java @@ -1,5 +1,6 @@ package com.datadog.iast.sink; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Location; import com.datadog.iast.model.Vulnerability; @@ -18,6 +19,10 @@ public class XContentTypeModuleImpl extends SinkModuleBase implements XContentTypeModule { private static final Logger LOGGER = LoggerFactory.getLogger(XContentTypeModuleImpl.class); + public XContentTypeModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onRequestEnd(final Object iastRequestContextObject, final IGSpanInfo igSpanInfo) { try { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XPathInjectionModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XPathInjectionModuleImpl.java index 7e501b969a6..31633777c5e 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XPathInjectionModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XPathInjectionModuleImpl.java @@ -2,6 +2,7 @@ import static com.datadog.iast.taint.Tainteds.canBeTainted; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.VulnerabilityType; import datadog.trace.api.iast.sink.XPathInjectionModule; @@ -10,6 +11,11 @@ import javax.annotation.Nullable; public class XPathInjectionModuleImpl extends SinkModuleBase implements XPathInjectionModule { + + public XPathInjectionModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onExpression(@Nullable String expression) { if (!canBeTainted(expression)) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XssModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XssModuleImpl.java index 0168b73924f..da3d1e18337 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XssModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/sink/XssModuleImpl.java @@ -3,6 +3,7 @@ import static com.datadog.iast.taint.Ranges.rangesProviderFor; import static com.datadog.iast.taint.Tainteds.canBeTainted; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Evidence; import com.datadog.iast.model.Location; @@ -21,6 +22,10 @@ public class XssModuleImpl extends SinkModuleBase implements XssModule { + public XssModuleImpl(final Dependencies dependencies) { + super(dependencies); + } + @Override public void onXss(@Nonnull String s) { if (!canBeTainted(s)) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/source/WebModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/source/WebModuleImpl.java deleted file mode 100644 index b036a740f06..00000000000 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/source/WebModuleImpl.java +++ /dev/null @@ -1,161 +0,0 @@ -package com.datadog.iast.source; - -import static com.datadog.iast.taint.Tainteds.canBeTainted; - -import com.datadog.iast.IastRequestContext; -import com.datadog.iast.model.Source; -import com.datadog.iast.taint.TaintedObjects; -import datadog.trace.api.iast.SourceTypes; -import datadog.trace.api.iast.source.WebModule; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.Map; -import javax.annotation.Nullable; - -public class WebModuleImpl implements WebModule { - - @Override - public void onParameterNames(@Nullable final Collection paramNames) { - onNamed(paramNames, SourceTypes.REQUEST_PARAMETER_NAME); - } - - @Override - public void onParameterValues( - @Nullable final String paramName, @Nullable final String[] paramValues) { - onNamed(paramName, paramValues, SourceTypes.REQUEST_PARAMETER_VALUE); - } - - @Override - public void onParameterValues( - @Nullable final String paramName, @Nullable final Collection paramValues) { - onNamed(paramName, paramValues, SourceTypes.REQUEST_PARAMETER_VALUE); - } - - @Override - public void onParameterValues(@Nullable final Map values) { - onNamed(values, SourceTypes.REQUEST_PARAMETER_VALUE); - } - - @Override - public void onHeaderNames(@Nullable final Collection headerNames) { - onNamed(headerNames, SourceTypes.REQUEST_HEADER_NAME); - } - - @Override - public void onHeaderValues( - @Nullable final String headerName, @Nullable final Collection headerValues) { - onNamed(headerName, headerValues, SourceTypes.REQUEST_HEADER_VALUE); - } - - @Override - public void onMultipartNames(@Nullable final Collection headerNames) { - onNamed(headerNames, SourceTypes.REQUEST_MULTIPART_PARAMETER); - } - - @Override - public void onMultipartValues( - @Nullable final String headerName, @Nullable final Collection headerValues) { - onNamed(headerName, headerValues, SourceTypes.REQUEST_MULTIPART_PARAMETER); - } - - @Override - public void onCookieNames(@Nullable Iterable cookieNames) { - onNamed(cookieNames, SourceTypes.REQUEST_COOKIE_NAME); - } - - @Override - public void onGetPathInfo(@Nullable String s) { - onNamed(Collections.singleton(s), SourceTypes.REQUEST_PATH); - } - - @Override - public void onGetRequestURI(@Nullable String s) { - onNamed(Collections.singleton(s), SourceTypes.REQUEST_URI); - } - - private static void onNamed(@Nullable final Iterable names, final byte source) { - if (names == null) { - return; - } - Iterator iterator = names.iterator(); - if (!iterator.hasNext()) { - return; - } - - final IastRequestContext ctx = IastRequestContext.get(); - if (ctx == null) { - return; - } - final TaintedObjects taintedObjects = ctx.getTaintedObjects(); - do { - String name = iterator.next(); - if (canBeTainted(name)) { - taintedObjects.taintInputString(name, new Source(source, name, name)); - } - } while (iterator.hasNext()); - } - - private static void onNamed( - @Nullable final String name, @Nullable final Iterable values, final byte source) { - if (values == null) { - return; - } - Iterator iterator = values.iterator(); - if (!iterator.hasNext()) { - return; - } - - final IastRequestContext ctx = IastRequestContext.get(); - if (ctx == null) { - return; - } - final TaintedObjects taintedObjects = ctx.getTaintedObjects(); - do { - String value = iterator.next(); - if (canBeTainted(value)) { - taintedObjects.taintInputString(value, new Source(source, name, value)); - } - } while (iterator.hasNext()); - } - - private static void onNamed( - @Nullable final String name, @Nullable final String[] values, final byte source) { - if (values == null || values.length == 0) { - return; - } - final IastRequestContext ctx = IastRequestContext.get(); - if (ctx == null) { - return; - } - final TaintedObjects taintedObjects = ctx.getTaintedObjects(); - for (final String value : values) { - if (canBeTainted(value)) { - taintedObjects.taintInputString(value, new Source(source, name, value)); - } - } - } - - private static void onNamed(@Nullable final Map values, final byte source) { - if (values == null || values.isEmpty()) { - return; - } - final IastRequestContext ctx = IastRequestContext.get(); - if (ctx == null) { - return; - } - final TaintedObjects taintedObjects = ctx.getTaintedObjects(); - final byte nameSource = SourceTypes.namedSource(source); - for (final Map.Entry entry : values.entrySet()) { - final String name = entry.getKey(); - if (canBeTainted(name)) { - taintedObjects.taintInputString(name, new Source(nameSource, name, name)); - } - for (final String value : entry.getValue()) { - if (canBeTainted(value)) { - taintedObjects.taintInputString(value, new Source(source, name, value)); - } - } - } - } -} diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Ranges.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Ranges.java index ce89da54002..992655086c5 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Ranges.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Ranges.java @@ -1,7 +1,7 @@ package com.datadog.iast.taint; -import static com.datadog.iast.model.Range.NOT_MARKED; import static com.datadog.iast.taint.TaintedObject.MAX_RANGE_COUNT; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import com.datadog.iast.model.Range; import com.datadog.iast.model.Source; @@ -41,11 +41,20 @@ public Range[] ranges(final Object value) { private Ranges() {} - public static Range[] forString( - final @Nonnull String obj, final @Nonnull Source source, final int mark) { + public static Range[] forCharSequence( + final @Nonnull CharSequence obj, final @Nonnull Source source) { + return forCharSequence(obj, source, NOT_MARKED); + } + + public static Range[] forCharSequence( + final @Nonnull CharSequence obj, final @Nonnull Source source, final int mark) { return new Range[] {new Range(0, obj.length(), source, mark)}; } + public static Range[] forObject(final @Nonnull Source source) { + return forObject(source, NOT_MARKED); + } + public static Range[] forObject(final @Nonnull Source source, final int mark) { return new Range[] {new Range(0, Integer.MAX_VALUE, source, mark)}; } @@ -76,7 +85,7 @@ public static void copyShift( public static Range[] mergeRanges( final int offset, @Nonnull final Range[] rangesLeft, @Nonnull final Range[] rangesRight) { - final long nRanges = rangesLeft.length + rangesRight.length; + final long nRanges = rangesLeft.length + (long) rangesRight.length; final Range[] ranges = newArray(nRanges); int remaining = ranges.length; if (rangesLeft.length > 0) { @@ -114,6 +123,7 @@ public static RangesProvider rangesProviderFor( return new ListProvider<>(items, to); } + @Nullable public static Range[] forSubstring(int offset, int length, final @Nonnull Range[] ranges) { int[] includedRangesInterval = getIncludedRangesInterval(offset, length, ranges); @@ -175,6 +185,7 @@ public static int[] getIncludedRangesInterval( return new int[] {start, end}; } + @Nonnull public static Range highestPriorityRange(@Nonnull final Range[] ranges) { /* * This approach is better but not completely correct ideally the highest priority should use the following patterns: @@ -204,14 +215,16 @@ public interface RangesProvider { int size(); + @Nullable E value(final int index); + @Nullable Range[] ranges(final E value); } private abstract static class IterableProvider implements RangesProvider { private final LIST items; - private final Map ranges; + @Nullable private final Map ranges; private final int rangeCount; private IterableProvider(@Nonnull final LIST items, @Nonnull final TaintedObjects to) { @@ -242,11 +255,13 @@ public int rangeCount() { return rangeCount; } + @Nullable @Override public E value(final int index) { return item(items, index); } + @Nullable @Override public Range[] ranges(final E value) { return ranges == null ? null : ranges.get(value); @@ -259,12 +274,13 @@ public int size() { protected abstract int size(@Nonnull final LIST items); + @Nullable protected abstract E item(@Nonnull final LIST items, final int index); } private static class SingleProvider implements RangesProvider { private final E value; - private final TaintedObject tainted; + @Nullable private final TaintedObject tainted; private SingleProvider(@Nonnull final E value, @Nonnull final TaintedObjects to) { this.value = value; @@ -281,11 +297,13 @@ public int size() { return 1; } + @Nullable @Override public E value(int index) { return index == 0 ? value : null; } + @Nullable @Override public Range[] ranges(E value) { return value == this.value && tainted != null ? tainted.getRanges() : null; @@ -303,6 +321,7 @@ protected int size(@Nonnull final E[] items) { return items.length; } + @Nullable @Override protected E item(@Nonnull final E[] items, final int index) { return items[index]; @@ -320,13 +339,15 @@ protected int size(@Nonnull final List items) { return items.size(); } + @Nullable @Override protected E item(@Nonnull final List items, final int index) { return items.get(index); } } - public static Range[] getNotMarkedRanges(final Range[] ranges, final int mark) { + @Nullable + public static Range[] getNotMarkedRanges(@Nullable final Range[] ranges, final int mark) { if (ranges == null) { return null; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedMap.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedMap.java index fa690a22ded..79f91c18a6a 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedMap.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedMap.java @@ -223,7 +223,7 @@ private int remove(final TaintedObject entry) { if (cur == null) { return 0; } - for (TaintedObject prev = cur.next; cur != null; prev = cur, cur = cur.next) { + for (TaintedObject prev = cur.next; cur != null && prev != null; prev = cur, cur = cur.next) { if (cur == entry) { prev.next = cur.next; return 1; @@ -259,7 +259,7 @@ private int index(int h) { private Iterator iterator(final int start, final int stop) { return new Iterator() { int currentIndex = start; - TaintedObject currentSubPos; + @Nullable TaintedObject currentSubPos; @Override public boolean hasNext() { @@ -294,6 +294,8 @@ public TaintedObject next() { }; } + @Nonnull + @Override public Iterator iterator() { return iterator(0, table.length); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObject.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObject.java index d8b60f6b68e..84d3b9c6091 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObject.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObject.java @@ -14,7 +14,7 @@ public class TaintedObject extends WeakReference { public static final int MAX_RANGE_COUNT = Config.get().getIastMaxRangeCount(); final int positiveHashCode; - TaintedObject next; + @Nullable TaintedObject next; private Range[] ranges; public TaintedObject( diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObjects.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObjects.java index e339e134edb..ab3b20e0401 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObjects.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/TaintedObjects.java @@ -1,13 +1,10 @@ package com.datadog.iast.taint; -import static com.datadog.iast.model.Range.NOT_MARKED; import static datadog.trace.api.ConfigDefaults.DEFAULT_IAST_MAX_CONCURRENT_REQUESTS; import static java.util.Collections.emptyIterator; -import com.datadog.iast.IastRequestContext; import com.datadog.iast.IastSystem; import com.datadog.iast.model.Range; -import com.datadog.iast.model.Source; import com.datadog.iast.model.json.TaintedObjectEncoding; import datadog.trace.api.Config; import java.util.ArrayList; @@ -16,17 +13,17 @@ import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings("UnusedReturnValue") public interface TaintedObjects extends Iterable { - TaintedObject taintInputString(@Nonnull String obj, @Nonnull Source source, int mark); - - TaintedObject taintInputObject(@Nonnull Object obj, @Nonnull Source source, int mark); - + @Nullable TaintedObject taint(@Nonnull Object obj, @Nonnull Range[] ranges); + @Nullable TaintedObject get(@Nonnull Object obj); void release(); @@ -37,14 +34,6 @@ public interface TaintedObjects extends Iterable { boolean isFlat(); - default TaintedObject taintInputString(@Nonnull String obj, @Nonnull Source source) { - return taintInputString(obj, source, NOT_MARKED); - } - - default TaintedObject taintInputObject(@Nonnull Object obj, @Nonnull Source source) { - return taintInputObject(obj, source, NOT_MARKED); - } - static TaintedObjects acquire() { TaintedObjectsImpl taintedObjects = TaintedObjectsImpl.pool.poll(); if (taintedObjects == null) { @@ -53,22 +42,6 @@ static TaintedObjects acquire() { return IastSystem.DEBUG ? new TaintedObjectsDebugAdapter(taintedObjects) : taintedObjects; } - static TaintedObjects activeTaintedObjects(boolean lazy) { - if (lazy) { - return new LazyTaintedObjects(); - } else { - final IastRequestContext ctx = IastRequestContext.get(); - if (ctx != null) { - return ctx.getTaintedObjects(); - } - return null; - } - } - - static TaintedObjects activeTaintedObjects() { - return activeTaintedObjects(false); - } - class TaintedObjectsImpl implements TaintedObjects { private static final ArrayBlockingQueue pool = @@ -86,24 +59,6 @@ private TaintedObjectsImpl(final @Nonnull TaintedMap map) { this.map = map; } - @Override - public TaintedObject taintInputString( - final @Nonnull String obj, final @Nonnull Source source, final int mark) { - final TaintedObject tainted = - new TaintedObject(obj, Ranges.forString(obj, source, mark), map.getReferenceQueue()); - map.put(tainted); - return tainted; - } - - @Override - public TaintedObject taintInputObject( - @Nonnull Object obj, @Nonnull Source source, final int mark) { - final TaintedObject tainted = - new TaintedObject(obj, Ranges.forObject(source, mark), map.getReferenceQueue()); - map.put(tainted); - return tainted; - } - @Override public TaintedObject taint(final @Nonnull Object obj, final @Nonnull Range[] ranges) { final TaintedObject tainted = new TaintedObject(obj, ranges, map.getReferenceQueue()); @@ -111,6 +66,7 @@ public TaintedObject taint(final @Nonnull Object obj, final @Nonnull Range[] ran return tainted; } + @Nullable @Override public TaintedObject get(final @Nonnull Object obj) { return map.get(obj); @@ -137,13 +93,14 @@ public boolean isFlat() { return map.isFlat(); } + @Nonnull @Override public Iterator iterator() { return map.iterator(); } } - class TaintedObjectsDebugAdapter implements TaintedObjects { + final class TaintedObjectsDebugAdapter implements TaintedObjects { static final Logger LOGGER = LoggerFactory.getLogger(TaintedObjects.class); private final TaintedObjectsImpl delegated; @@ -155,22 +112,7 @@ public TaintedObjectsDebugAdapter(final TaintedObjectsImpl delegated) { LOGGER.debug("new: id={}", id); } - @Override - public TaintedObject taintInputString( - final @Nonnull String obj, final @Nonnull Source source, final int mark) { - final TaintedObject tainted = delegated.taintInputString(obj, source, mark); - logTainted(tainted); - return tainted; - } - - @Override - public TaintedObject taintInputObject( - @Nonnull Object obj, @Nonnull Source source, final int mark) { - final TaintedObject tainted = delegated.taintInputObject(obj, source, mark); - logTainted(tainted); - return tainted; - } - + @Nullable @Override public TaintedObject taint(final @Nonnull Object obj, final @Nonnull Range[] ranges) { final TaintedObject tainted = delegated.taint(obj, ranges); @@ -178,6 +120,7 @@ public TaintedObject taint(final @Nonnull Object obj, final @Nonnull Range[] ran return tainted; } + @Nullable @Override public TaintedObject get(final @Nonnull Object obj) { return delegated.get(obj); @@ -214,6 +157,7 @@ public boolean isFlat() { return delegated.isFlat(); } + @Nonnull @Override public Iterator iterator() { return delegated.iterator(); @@ -230,97 +174,17 @@ private void logTainted(final TaintedObject tainted) { } } - class LazyTaintedObjects implements TaintedObjects { - private boolean fetched = false; - private TaintedObjects taintedObjects; - - @Override - public TaintedObject taintInputString( - @Nonnull final String obj, @Nonnull final Source source, final int mark) { - final TaintedObjects to = getTaintedObjects(); - return to == null ? null : to.taintInputString(obj, source, mark); - } - - @Override - public TaintedObject taintInputObject( - @Nonnull final Object obj, @Nonnull final Source source, final int mark) { - final TaintedObjects to = getTaintedObjects(); - return to == null ? null : to.taintInputObject(obj, source, mark); - } - - @Override - public TaintedObject taint(@Nonnull final Object obj, @Nonnull final Range[] ranges) { - final TaintedObjects to = getTaintedObjects(); - return to == null ? null : to.taint(obj, ranges); - } - - @Override - public TaintedObject get(@Nonnull final Object obj) { - final TaintedObjects to = getTaintedObjects(); - return to == null ? null : to.get(obj); - } - - @Override - public void release() { - final TaintedObjects to = getTaintedObjects(); - if (to != null) { - to.release(); - } - } - - @Override - public Iterator iterator() { - final TaintedObjects to = getTaintedObjects(); - return to != null ? to.iterator() : emptyIterator(); - } - - @Override - public int getEstimatedSize() { - final TaintedObjects to = getTaintedObjects(); - return to != null ? to.getEstimatedSize() : 0; - } - - @Override - public boolean isFlat() { - final TaintedObjects to = getTaintedObjects(); - return to != null && to.isFlat(); - } - - @Override - public int count() { - final TaintedObjects to = getTaintedObjects(); - return to != null ? to.count() : 0; - } - - private TaintedObjects getTaintedObjects() { - if (!fetched) { - fetched = true; - taintedObjects = activeTaintedObjects(); - } - return taintedObjects; - } - } - - enum NoOp implements TaintedObjects { - INSTANCE; - - @Override - public TaintedObject taintInputString( - @Nonnull final String obj, @Nonnull final Source source, final int mark) { - return null; - } + final class NoOp implements TaintedObjects { - @Override - public TaintedObject taintInputObject( - @Nonnull final Object obj, @Nonnull final Source source, final int mark) { - return null; - } + public static final TaintedObjects INSTANCE = new NoOp(); + @Nullable @Override public TaintedObject taint(@Nonnull final Object obj, @Nonnull final Range[] ranges) { return null; } + @Nullable @Override public TaintedObject get(@Nonnull final Object obj) { return null; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Tainteds.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Tainteds.java index 142249ec1fe..422f677d553 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Tainteds.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/taint/Tainteds.java @@ -1,17 +1,21 @@ package com.datadog.iast.taint; import java.util.Collection; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.jetbrains.annotations.Contract; /** Utilitiles to work with {@link TaintedObject} */ public final class Tainteds { private Tainteds() {} + @Contract("null -> false") public static boolean canBeTainted(@Nullable final CharSequence s) { return s != null && s.length() > 0; } + @Contract("null -> false") public static boolean canBeTainted(@Nullable final E[] e) { if (e == null || e.length == 0) { return false; @@ -24,6 +28,7 @@ public static boolean canBeTainted(@Nullable final E[] return false; } + @Contract("null -> false") public static boolean canBeTainted(@Nullable final Collection e) { if (e == null || e.isEmpty()) { return false; @@ -36,7 +41,9 @@ public static boolean canBeTainted(@Nullable final Coll return false; } - public static TaintedObject getTainted(final TaintedObjects to, final Object value) { + @Nullable + public static TaintedObject getTainted( + @Nonnull final TaintedObjects to, @Nullable final Object value) { return value == null ? null : to.get(value); } } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/TelemetryRequestStartedHandler.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/TelemetryRequestStartedHandler.java index 1bf6033ca8a..5e647355b09 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/TelemetryRequestStartedHandler.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/TelemetryRequestStartedHandler.java @@ -1,6 +1,6 @@ package com.datadog.iast.telemetry; -import com.datadog.iast.HasDependencies.Dependencies; +import com.datadog.iast.Dependencies; import com.datadog.iast.IastRequestContext; import com.datadog.iast.RequestStartedHandler; import com.datadog.iast.taint.TaintedObjects; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetry.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetry.java index c86450230f7..1ac059b0b54 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetry.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetry.java @@ -6,13 +6,13 @@ import com.datadog.iast.IastRequestContext; import com.datadog.iast.model.Range; -import com.datadog.iast.model.Source; import com.datadog.iast.taint.TaintedObject; import com.datadog.iast.taint.TaintedObjects; import datadog.trace.api.iast.telemetry.IastMetricCollector; import datadog.trace.api.iast.telemetry.Verbosity; import java.util.Iterator; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class TaintedObjectsWithTelemetry implements TaintedObjects { @@ -31,7 +31,7 @@ public static TaintedObjects build( private final TaintedObjects delegate; private final boolean debug; - private IastRequestContext ctx; + @Nullable private IastRequestContext ctx; protected TaintedObjectsWithTelemetry(final boolean debug, final TaintedObjects delegate) { this.delegate = delegate; @@ -46,16 +46,7 @@ public void initContext(final IastRequestContext ctx) { this.ctx = ctx; } - @Override - public TaintedObject taintInputString( - @Nonnull String obj, @Nonnull Source source, final int mark) { - final TaintedObject result = delegate.taintInputString(obj, source, mark); - if (debug) { - IastMetricCollector.add(EXECUTED_TAINTED, 1, ctx); - } - return result; - } - + @Nullable @Override public TaintedObject taint(@Nonnull Object obj, @Nonnull Range[] ranges) { final TaintedObject result = delegate.taint(obj, ranges); @@ -65,16 +56,7 @@ public TaintedObject taint(@Nonnull Object obj, @Nonnull Range[] ranges) { return result; } - @Override - public TaintedObject taintInputObject( - @Nonnull Object obj, @Nonnull Source source, final int mark) { - final TaintedObject result = delegate.taintInputObject(obj, source, mark); - if (debug) { - IastMetricCollector.add(EXECUTED_TAINTED, 1, ctx); - } - return result; - } - + @Nullable @Override public TaintedObject get(@Nonnull Object obj) { return delegate.get(obj); @@ -92,6 +74,7 @@ public void release() { } } + @Nonnull @Override public Iterator iterator() { return delegate.iterator(); diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/CookieSecurityParser.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/CookieSecurityParser.java index 54dec1ec58a..bc238eac288 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/CookieSecurityParser.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/CookieSecurityParser.java @@ -5,8 +5,10 @@ import datadog.trace.api.iast.util.Cookie; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.NoSuchElementException; import java.util.StringTokenizer; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +46,7 @@ public static List parse(final String cookieString) { } } + @Nullable private static Cookie parseInternal(final String header) { String cookieName; boolean httpOnly = false; @@ -111,7 +114,7 @@ private static List splitMultiCookies(final String header) { } private static int guessCookieVersion(String header) { - header = header.toLowerCase(); + header = header.toLowerCase(Locale.ROOT); if (header.contains("expires=")) { // only netscape cookie using 'expires' return 0; diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/HttpHeader.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/HttpHeader.java index 36ee2b86501..5d58ea8a617 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/HttpHeader.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/HttpHeader.java @@ -6,6 +6,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nullable; public class HttpHeader { @@ -19,6 +20,7 @@ public boolean matches(final String name) { return this.name.equalsIgnoreCase(name); } + @Nullable public static HttpHeader from(final String name) { return Values.HEADERS.get(name.toLowerCase(Locale.ROOT)); } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/ObjectVisitor.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/ObjectVisitor.java index c9178b749c1..01f5e568061 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/ObjectVisitor.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/ObjectVisitor.java @@ -15,6 +15,7 @@ import java.util.Set; import java.util.function.Predicate; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +26,7 @@ public class ObjectVisitor { private static final int MAX_VISITED_OBJECTS = 1000; private static final int MAX_DEPTH = 10; - private static final Method TRY_SET_ACCESSIBLE; + @Nullable private static final Method TRY_SET_ACCESSIBLE; static { TRY_SET_ACCESSIBLE = fetchTrySetAccessibleMethod(); @@ -210,6 +211,7 @@ private static boolean inspectField(final Field field) { return true; } + @Nullable private static Method fetchTrySetAccessibleMethod() { Method method = null; if (Platform.isJavaVersionAtLeast(9)) { diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/Ranged.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/Ranged.java index 1cde27dc057..645fb58cfdc 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/Ranged.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/Ranged.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; public interface Ranged { @@ -62,6 +63,7 @@ default List remove(final Ranged range) { } /** Computes the intersection of the ranges or {@code null} if they do not intersect */ + @Nullable default Ranged intersection(final Ranged range) { if (this.getStart() == range.getStart() && this.getLength() == range.getLength()) { return this; @@ -84,7 +86,7 @@ default Ranged intersection(final Ranged range) { } } - default boolean isBefore(final Ranged range) { + default boolean isBefore(@Nullable final Ranged range) { if (range == null) { return true; } diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/RangedDeque.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/RangedDeque.java index faef93afde0..4e66a2bf5a1 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/RangedDeque.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/util/RangedDeque.java @@ -9,8 +9,10 @@ /** */ public interface RangedDeque { + @Nullable E poll(); + @Nullable E peek(); void addFirst(@Nonnull E item); @@ -31,8 +33,9 @@ abstract class BaseRangedDequeue implements RangedDeque { private final Deque head = new LinkedList<>(); - protected E next; + @Nullable protected E next; + @Nullable @Override public final E poll() { final E result = next; @@ -40,6 +43,7 @@ public final E poll() { return result; } + @Nullable @Override public final E peek() { return next; @@ -58,15 +62,18 @@ public final boolean isEmpty() { return next == null; } + @Nullable protected final E fetchNext() { return head.isEmpty() ? internalPoll() : head.poll(); } + @Nullable protected abstract E internalPoll(); } class EmptyRangedDequeue extends BaseRangedDequeue { + @Nullable @Override protected E internalPoll() { return null; @@ -82,6 +89,7 @@ class TokenizerQueue extends BaseRangedDequeue { next = fetchNext(); } + @Nullable @Override protected Ranged internalPoll() { return tokenizer.next() ? tokenizer.current() : null; @@ -99,6 +107,7 @@ class ArrayQueue extends BaseRangedDequeue { next = fetchNext(); } + @Nullable @Override protected E internalPoll() { return index >= array.length ? null : array[index++]; diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/GrpcRequestMessageHandlerTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/GrpcRequestMessageHandlerTest.groovy index d3c31196f87..e4081f96d7d 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/GrpcRequestMessageHandlerTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/GrpcRequestMessageHandlerTest.groovy @@ -62,7 +62,7 @@ class GrpcRequestMessageHandlerTest extends DDSpecification { handler.apply(ctx, target) then: - 1 * propagation.taintDeeply(iastCtx, SourceTypes.GRPC_BODY, target, _ as Predicate>) + 1 * propagation.taintDeeply(iastCtx, target, SourceTypes.GRPC_BODY, _ as Predicate>) } void 'the handler only takes into account protobuf v.#protobufVersion related messages'() { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/IastModuleImplTestBase.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/IastModuleImplTestBase.groovy index d47ce8e0a14..9ad167ed6c6 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/IastModuleImplTestBase.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/IastModuleImplTestBase.groovy @@ -1,6 +1,5 @@ package com.datadog.iast -import com.datadog.iast.HasDependencies.Dependencies import com.datadog.iast.overhead.Operation import com.datadog.iast.overhead.OverheadController import datadog.trace.api.Config @@ -25,6 +24,8 @@ class IastModuleImplTestBase extends DDSpecification { // TODO replace by mock an fix all mock assertions (0 * _ will usually fail) protected StackWalker stackWalker = StackWalkerFactory.INSTANCE + protected Dependencies dependencies = new Dependencies(Config.get(), reporter, overheadController, stackWalker) + void setup() { AgentTracer.forceRegister(tracer) overheadController.acquireRequest() >> true @@ -34,9 +35,4 @@ class IastModuleImplTestBase extends DDSpecification { void cleanup() { AgentTracer.forceRegister(ORIGINAL_TRACER) } - - protected E registerDependencies(final E module) { - module.registerDependencies(new Dependencies(Config.get(), reporter, overheadController, stackWalker)) - return module - } } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestEndedHandlerTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestEndedHandlerTest.groovy index b205d9d3eba..b325a373d90 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestEndedHandlerTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestEndedHandlerTest.groovy @@ -1,6 +1,5 @@ package com.datadog.iast -import com.datadog.iast.HasDependencies.Dependencies import com.datadog.iast.overhead.OverheadController import datadog.trace.api.Config import datadog.trace.api.gateway.Flow diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestStartedHandlerTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestStartedHandlerTest.groovy index f70e087b146..8a0ca4e2c4a 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestStartedHandlerTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/RequestStartedHandlerTest.groovy @@ -1,6 +1,5 @@ package com.datadog.iast -import com.datadog.iast.HasDependencies.Dependencies import com.datadog.iast.overhead.OverheadController import datadog.trace.api.Config import datadog.trace.api.gateway.Flow diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/RangeTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/RangeTest.groovy index 5a3561fe7f8..c4992278756 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/RangeTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/RangeTest.groovy @@ -35,7 +35,7 @@ class RangeTest extends DDSpecification { def 'shift zero'() { given: final source = new Source(SourceTypes.NONE, null, null) - final orig = new Range(0, 1, source, Range.NOT_MARKED) + final orig = new Range(0, 1, source, VulnerabilityMarks.NOT_MARKED) when: final result = orig.shift(0) diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/VulnerabilityTypeTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/VulnerabilityTypeTest.groovy index 51568f75e64..408219c9000 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/VulnerabilityTypeTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/VulnerabilityTypeTest.groovy @@ -27,9 +27,6 @@ class VulnerabilityTypeTest extends DDSpecification { WEAK_CIPHER | getSpanAndClassAndMethodLocation(123) | new Evidence("MD5") | 3265519776 WEAK_CIPHER | getSpanAndClassAndMethodLocation(456) | new Evidence("MD4") | 3265519776 WEAK_CIPHER | getSpanAndClassAndMethodLocation(789) | null | 3265519776 - WEAK_CIPHER | null | new Evidence("MD5") | 1272119222 - WEAK_CIPHER | null | new Evidence("MD4") | 1272119222 - WEAK_CIPHER | null | null | 1272119222 INSECURE_COOKIE | getSpanAndStackLocation(123) | null | 3471934557 INSECURE_COOKIE | getSpanAndStackLocation(123) | new Evidence("cookieName1") | 360083726 INSECURE_COOKIE | getSpanAndStackLocation(123) | new Evidence("cookieName2") | 2357141684 @@ -47,19 +44,19 @@ class VulnerabilityTypeTest extends DDSpecification { HSTS_HEADER_MISSING | getSpanLocation(123, 'serviceName2') | null | 1268102093 } - private Location getSpanAndStackLocation(final long spanId){ + private Location getSpanAndStackLocation(final long spanId) { final span = Mock(AgentSpan) span.getSpanId() >> spanId return Location.forSpanAndStack(span, new StackTraceElement("foo", "foo", "foo", 1)) } - private Location getSpanAndClassAndMethodLocation(final long spanId){ + private Location getSpanAndClassAndMethodLocation(final long spanId) { final span = Mock(AgentSpan) span.getSpanId() >> spanId return Location.forSpanAndClassAndMethod(span, "foo", "foo") } - private Location getSpanLocation(final long spanId, final String serviceName){ + private Location getSpanLocation(final long spanId, final String serviceName) { final span = Mock(AgentSpan) span.getSpanId() >> spanId span.getServiceName() >> serviceName diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceEncodingTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceEncodingTest.groovy index c946e53f70a..9b3d67e9339 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceEncodingTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceEncodingTest.groovy @@ -10,6 +10,8 @@ import datadog.trace.test.util.DDSpecification import org.skyscreamer.jsonassert.JSONAssert import spock.lang.Shared +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED + class EvidenceEncodingTest extends DDSpecification { private static final List SOURCES_SUITE = (0..2).collect { new Source((byte) it, "name$it", "value$it") } @@ -65,7 +67,7 @@ class EvidenceEncodingTest extends DDSpecification { } private static Range range(final int start, final int length, final Source source) { - return new Range(start, length, source, Range.NOT_MARKED) + return new Range(start, length, source, NOT_MARKED) } private static Source source(final int index) { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/TaintedObjectEncodingTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/TaintedObjectEncodingTest.groovy index 55956358225..dfbc0fb8275 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/TaintedObjectEncodingTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/TaintedObjectEncodingTest.groovy @@ -10,6 +10,8 @@ import org.skyscreamer.jsonassert.JSONAssert import java.lang.ref.ReferenceQueue +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED + class TaintedObjectEncodingTest extends DDSpecification { @Override @@ -87,7 +89,7 @@ class TaintedObjectEncodingTest extends DDSpecification { private TaintedObject taintedObject(final String value, final byte sourceType, final String sourceName, final String sourceValue) { return new TaintedObject( value, - [new Range(0, value.length(), new Source(sourceType, sourceName, sourceValue), Range.NOT_MARKED)] as Range[], + [new Range(0, value.length(), new Source(sourceType, sourceName, sourceValue), NOT_MARKED)] as Range[], Mock(ReferenceQueue)) } } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy index 1b4518fa1a0..4ade1c08a8d 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy @@ -13,7 +13,10 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan import datadog.trace.test.util.DDSpecification import org.skyscreamer.jsonassert.JSONAssert -import static com.datadog.iast.model.Range.NOT_MARKED +import java.util.regex.Matcher +import java.util.regex.Pattern + +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED class VulnerabilityEncodingTest extends DDSpecification { @@ -28,7 +31,15 @@ class VulnerabilityEncodingTest extends DDSpecification { value.add(null) when: - final result = VulnerabilityEncoding.toJson(value) + def result = VulnerabilityEncoding.toJson(value) + + then: + JSONAssert.assertEquals('''{ + "vulnerabilities": [] + }''', result, true) + + when: + result = VulnerabilityEncoding.getExceededTagSizeJson(new TruncatedVulnerabilities(value.getVulnerabilities())) then: JSONAssert.assertEquals('''{ @@ -420,4 +431,162 @@ class VulnerabilityEncodingTest extends DDSpecification { ] }''', result, true) } + + void 'one truncated vulnerability'() { + given: + final span = Mock(AgentSpan) + final spanId = 123456 + span.getSpanId() >> spanId + final value = new VulnerabilityBatch() + value.add(new Vulnerability( + VulnerabilityType.WEAK_HASH, + Location.forSpanAndStack(span, new StackTraceElement("foo", "fooMethod", "foo", 1)), + new Evidence(generateLargeString()) + )) + + when: + final result = VulnerabilityEncoding.getExceededTagSizeJson(new TruncatedVulnerabilities(value.getVulnerabilities())) + + then: + JSONAssert.assertEquals('''{ + "vulnerabilities": [ + { + "type": "WEAK_HASH", + "evidence": { + "value": "MAX SIZE EXCEEDED" + }, + "hash":1042880134, + "location": { + "spanId": 123456, + "line": 1, + "method": "fooMethod", + "path": "foo" + } + } + ] + }''', result, true) + } + + void 'two truncated vulnerabilities'() { + given: + final span = Mock(AgentSpan) + final spanId = 123456 + span.getSpanId() >> spanId + final value = new VulnerabilityBatch() + value.add(new Vulnerability( + VulnerabilityType.WEAK_HASH, + Location.forSpanAndStack(span, new StackTraceElement("foo", "fooMethod", "foo", 1)), + new Evidence(generateLargeString(), [new Range(0, 1, new Source(SourceTypes.REQUEST_PARAMETER_VALUE, "key1", "value"), NOT_MARKED)] as Range[]) + )) + value.add(new Vulnerability( + VulnerabilityType.WEAK_HASH, + Location.forSpanAndStack(span, new StackTraceElement("foo", "fooMethod", "foo", 1)), + new Evidence(generateLargeString(), [new Range(0, 1, new Source(SourceTypes.REQUEST_PARAMETER_VALUE, "key2", "value"), NOT_MARKED)] as Range[]) + )) + + when: + final result = VulnerabilityEncoding.getExceededTagSizeJson(new TruncatedVulnerabilities(value.getVulnerabilities())) + + then: + JSONAssert.assertEquals('''{ + "vulnerabilities": [ + { + "evidence": { + "value": "MAX SIZE EXCEEDED" + }, + "hash": 1042880134, + "location": { + "spanId": 123456, + "line": 1, + "method": "fooMethod", + "path": "foo" + }, + "type": "WEAK_HASH" + }, + { + "evidence": { + "value": "MAX SIZE EXCEEDED" + }, + "hash": 1042880134, + "location": { + "spanId": 123456, + "line": 1, + "method": "fooMethod", + "path": "foo" + }, + "type": "WEAK_HASH" + } + ] + }''', result, true) + } + + void 'when json is greater than 25kb VulnerabilityEncoding#getExceededTagSizeJson is called'(){ + given: + final span = Mock(AgentSpan) + final spanId = 123456 + span.getSpanId() >> spanId + final value = new VulnerabilityBatch() + for (int i = 0 ; i < 40; i++){ + value.add(generateBigVulnerability(span)) + } + + when: + final result = VulnerabilityEncoding.toJson(value) + + then: 'all sources have been removed and all vulnerabilities have generic evidence' + !result.contains("source") //sources have been removed + countGenericEvidenceOccurrences(result) == value.getVulnerabilities().size() //All the vulnerabilities have generic evidence + + } + + void 'exception during serialization is caught'() { + given: + final value = new VulnerabilityBatch() + final type = Mock(VulnerabilityType) { + name() >> { throw new RuntimeException("ERROR") } + } + final vuln = new Vulnerability(type, null, null) + value.add(vuln) + + when: + final result = VulnerabilityEncoding.toJson(value) + + then: + JSONAssert.assertEquals('''{ + "vulnerabilities": [ + ] + }''', result, true) + } + + private static String generateLargeString(){ + int targetSize = 25 * 1024 + StringBuilder sb = new StringBuilder() + Random random = new Random() + while (sb.length() < targetSize){ + sb.append(random.nextInt()) + } + return sb.toString() + } + + + private static Vulnerability generateBigVulnerability(AgentSpan span){ + String largeString = generateLargeString() + return new Vulnerability( + VulnerabilityType.WEAK_HASH, + Location.forSpanAndStack(span, new StackTraceElement("foo", "fooMethod", "foo", 1)), + new Evidence(largeString, [ + new Range(0, largeString.length(), new Source(SourceTypes.REQUEST_PARAMETER_VALUE, "key2", largeString), NOT_MARKED) + ] as Range[]) + ) + } + + private static int countGenericEvidenceOccurrences(final String input){ + Pattern pattern = Pattern.compile("\"evidence\":\\{\"value\":\"MAX SIZE EXCEEDED\"}") + Matcher matcher = pattern.matcher(input) + int count = 0 + while (matcher.find()){ + count++ + } + return count + } } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/PropagationModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/PropagationModuleTest.groovy index c8b45e183ec..e573d5dfb2c 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/PropagationModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/PropagationModuleTest.groovy @@ -4,6 +4,9 @@ import com.datadog.iast.IastModuleImplTestBase import com.datadog.iast.IastRequestContext import com.datadog.iast.model.Range import com.datadog.iast.model.Source +import com.datadog.iast.taint.Ranges +import com.datadog.iast.taint.TaintedObject +import datadog.trace.api.Config import datadog.trace.api.gateway.RequestContext import datadog.trace.api.gateway.RequestContextSlot import datadog.trace.api.iast.SourceTypes @@ -11,25 +14,19 @@ import datadog.trace.api.iast.Taintable import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.propagation.PropagationModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import groovy.transform.CompileDynamic +import org.junit.Assume -import static com.datadog.iast.model.Range.NOT_MARKED -import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat -import static com.datadog.iast.taint.TaintUtils.fromTaintFormat -import static datadog.trace.api.iast.VulnerabilityMarks.SQL_INJECTION_MARK -import static datadog.trace.api.iast.VulnerabilityMarks.XPATH_INJECTION_MARK -import static datadog.trace.api.iast.VulnerabilityMarks.XSS_MARK +import static com.datadog.iast.taint.Ranges.highestPriorityRange +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED -@CompileDynamic class PropagationModuleTest extends IastModuleImplTestBase { private PropagationModule module - private List objectHolder + private IastRequestContext ctx - def setup() { + void setup() { module = new PropagationModuleImpl() - objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { getData(RequestContextSlot.IAST) >> ctx @@ -40,54 +37,49 @@ class PropagationModuleTest extends IastModuleImplTestBase { tracer.activeSpan() >> span } - void '#method null or empty'() { - when: + void '#method(#args) not taintable'() { + when: 'there is no context by default' module.&"$method".call(args.toArray()) - then: + then: 'no mock calls should happen' + 0 * _ + + when: 'there is a context' + args.add(0, ctx) + module.&"$method".call(args.toArray()) + + then: 'no mock calls should happen' 0 * _ where: - method | args - 'taintIfInputIsTainted' | [null, null] - 'taintIfInputIsTainted' | [null, new Object()] - 'taintIfInputIsTainted' | [null, 'test'] - 'taintIfInputIsTainted' | [new Object(), null] - 'taintIfInputIsTainted' | [null as String, null] - 'taintIfInputIsTainted' | ['', null] - 'taintIfInputIsTainted' | ['', new Object()] - 'taintIfInputIsTainted' | [null as String, new Object()] - 'taintIfInputIsTainted' | ['test', null] - 'taintIfInputIsTainted' | [null as String, 'test'] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', null as String, null] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', '', null] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', '', new Object()] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', null as String, new Object()] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'test', null] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', null as String, 'test'] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, [].toSet(), 'test'] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, ['test'].toSet(), null] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, [:].entrySet().toList(), 'test'] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, [key: "value"].entrySet().toList(), null] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', null as Collection, 'test'] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', [], 'test'] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', ['value'], null] - 'taintObjectIfInputIsTaintedKeepingRanges' | [null, new Object()] - 'taintObjectIfInputIsTaintedKeepingRanges' | [new Object(), null] - 'taintIfAnyInputIsTainted' | [null, null] - 'taintIfAnyInputIsTainted' | [null, [].toArray()] - 'taintIfAnyInputIsTainted' | ['test', [].toArray()] - 'taint' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', null as String] - 'taint' | [SourceTypes.REQUEST_PARAMETER_VALUE, null as String, null as String] - 'taint' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', ''] - 'taintObjects' | [SourceTypes.REQUEST_PARAMETER_VALUE, null as Object[]] - 'taintObjects' | [SourceTypes.REQUEST_PARAMETER_VALUE, [] as Object[]] - 'taintObjects' | [SourceTypes.REQUEST_PARAMETER_VALUE, null as Collection] - 'taintIfInputIsTaintedWithMarks' | ['', null, VulnerabilityMarks.XSS_MARK] - 'taintIfInputIsTaintedWithMarks' | ['', new Object(), VulnerabilityMarks.XSS_MARK] - 'taintIfInputIsTaintedWithMarks' | [null as String, new Object(), VulnerabilityMarks.XSS_MARK] - 'taintIfInputIsTaintedWithMarks' | ['test', null, VulnerabilityMarks.XSS_MARK] - 'taintIfInputIsTaintedWithMarks' | [null as String, 'test', VulnerabilityMarks.XSS_MARK] + method | args + 'taint' | [null, SourceTypes.REQUEST_PARAMETER_VALUE] + 'taint' | [null, SourceTypes.REQUEST_PARAMETER_VALUE, 'name'] + 'taint' | [null, SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value'] + 'taintIfTainted' | [null, 'test'] + 'taintIfTainted' | ['test', null] + 'taintIfTainted' | [null, 'test', false, NOT_MARKED] + 'taintIfTainted' | ['test', null, false, NOT_MARKED] + 'taintIfTainted' | [null, 'test'] + 'taintIfTainted' | ['test', null] + 'taintIfTainted' | [null, 'test', SourceTypes.REQUEST_PARAMETER_VALUE] + 'taintIfTainted' | ['test', null, SourceTypes.REQUEST_PARAMETER_VALUE] + 'taintIfTainted' | [null, 'test', SourceTypes.REQUEST_PARAMETER_VALUE, 'name'] + 'taintIfTainted' | ['test', null, SourceTypes.REQUEST_PARAMETER_VALUE, 'name'] + 'taintIfTainted' | [null, 'test', SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value'] + 'taintIfTainted' | ['test', null, SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value'] + 'taintIfAnyTainted' | [null, ['test'] as Object[]] + 'taintIfAnyTainted' | ['test', null] + 'taintIfAnyTainted' | ['test', [] as Object[]] + 'taintDeeply' | [ + null, + SourceTypes.REQUEST_PARAMETER_VALUE, + { + true + } + ] + 'findSource' | [null] + 'isTainted' | [null] } void '#method without span'() { @@ -99,379 +91,264 @@ class PropagationModuleTest extends IastModuleImplTestBase { 0 * _ where: - method | args - 'taintIfInputIsTainted' | [new Object(), new Object()] - 'taintIfInputIsTainted' | [new Object(), 'test'] - 'taintIfInputIsTainted' | ['test', new Object()] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value', new Object()] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', ['value'], new Object()] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, ['value'].toSet(), new Object()] - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, [key: 'value'].entrySet().toList(), new Object()] - 'taintObjectIfInputIsTaintedKeepingRanges' | [new Object(), new Object()] - 'taintIfAnyInputIsTainted' | ['value', ['test', 'test2'].toArray()] - 'taint' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value'] - 'taintObjects' | [SourceTypes.REQUEST_PARAMETER_VALUE, [new Object()] as Object[]] - 'taintObjects' | [SourceTypes.REQUEST_PARAMETER_VALUE, [new Object()]] - 'taintObjects' | [SourceTypes.REQUEST_PARAMETER_VALUE, [new Object()] as Collection] - 'taintIfInputIsTaintedWithMarks' | ['test', new Object(), VulnerabilityMarks.XSS_MARK] + method | args + 'taint' | ['test', SourceTypes.REQUEST_PARAMETER_VALUE] + 'taint' | ['test', SourceTypes.REQUEST_PARAMETER_VALUE, 'name'] + 'taint' | ['test', SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value'] + 'taintIfTainted' | ['test', 'test'] + 'taintIfTainted' | ['test', 'test', false, NOT_MARKED] + 'taintIfTainted' | ['test', 'test', SourceTypes.REQUEST_PARAMETER_VALUE] + 'taintIfTainted' | ['test', 'test', SourceTypes.REQUEST_PARAMETER_VALUE, 'name'] + 'taintIfTainted' | ['test', 'test', SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value'] + 'taintIfAnyTainted' | ['test', ['test']] + 'taintDeeply' | [ + 'test', + SourceTypes.REQUEST_PARAMETER_VALUE, + { + true + } + ] + 'findSource' | ['test'] + 'isTainted' | ['test'] } - void 'test propagation for #method'() { + void 'test taint'() { given: - final toTaint = toTaintClosure.call(args) - final targetMethod = module.&"$method" - final arguments = args.toArray() - final input = inputClosure.call(arguments) + final value = (target instanceof CharSequence) ? target.toString() : null + final source = taintedSource(value) + final ranges = Ranges.forObject(source) when: - targetMethod.call(arguments) + module.taint(target, source.origin, source.name, source.value) then: - assertNotTainted(toTaint) - - when: - taint(input) - targetMethod.call(arguments) - - then: - assertTainted(toTaint) + final tainted = getTaintedObject(target) + if (shouldTaint) { + assertTainted(tainted, ranges) + } else { + assert tainted == null + } where: - method | args | toTaintClosure | inputClosure - 'taintIfInputIsTainted' | [new Object(), 'I am an string'] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTainted' | [new Object(), new Object()] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTainted' | [new Object(), new MockTaintable()] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTainted' | ['Hello', 'I am an string'] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTainted' | ['Hello', new Object()] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTainted' | ['Hello', new MockTaintable()] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value', 'I am an string'] | { - it[2] - } | { - it[3] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value', new Object()] | { - it[2] - } | { - it[3] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value', new MockTaintable()] | { - it[2] - } | { - it[3] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', ['value'], 'I am an string'] | { - it[2][0] - } | { - it[3] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', ['value'], new Object()] | { - it[2][0] - } | { - it[3] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, 'name', ['value'], new MockTaintable()] | { - it[2][0] - } | { - it[3] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, ['value'].toSet(), 'I am an string'] | { - it[1][0] - } | { - it[2] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, ['value'].toSet(), new Object()] | { - it[1][0] - } | { - it[2] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, ['value'].toSet(), new MockTaintable()] | { - it[1][0] - } | { - it[2] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, [name: 'value'].entrySet().toList(), 'I am an string'] | { - it[1][0].value - } | { - it[2] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, [name: 'value'].entrySet().toList(), new Object()] | { - it[1][0].value - } | { - it[2] - } - 'taintIfInputIsTainted' | [SourceTypes.REQUEST_PARAMETER_VALUE, [name: 'value'].entrySet().toList(), new MockTaintable()] | { - it[1][0].value - } | { - it[2] - } - 'taintObjectIfInputIsTaintedKeepingRanges' | [new Object(), new Object()] | { - it[0] - } | { - it[1] - } - 'taintObjectIfInputIsTaintedKeepingRanges' | [new Object(), new MockTaintable()] | { - it[0] - } | { - it[1] - } - 'taintIfAnyInputIsTainted' | [new Object(), ['I am an string'].toArray()] | { - it[0] - } | { - it[1][0] - } - 'taintIfAnyInputIsTainted' | [new Object(), [new Object()].toArray()] | { - it[0] - } | { - it[1][0] - } - 'taintIfAnyInputIsTainted' | [new Object(), [new MockTaintable()].toArray()] | { - it[0] - } | { - it[1][0] - } - 'taintIfAnyInputIsTainted' | ['Hello', ['I am an string'].toArray()] | { - it[0] - } | { - it[1][0] - } - 'taintIfAnyInputIsTainted' | ['Hello', [new Object()].toArray()] | { - it[0] - } | { - it[1][0] - } - 'taintIfAnyInputIsTainted' | ['Hello', [new MockTaintable()].toArray()] | { - it[0] - } | { - it[1][0] - } - 'taintIfInputIsTaintedWithMarks' | ['Hello', 'I am an string', VulnerabilityMarks.XSS_MARK] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTaintedWithMarks' | ['Hello', new Object(), VulnerabilityMarks.XSS_MARK] | { - it[0] - } | { - it[1] - } - 'taintIfInputIsTaintedWithMarks' | ['Hello', new MockTaintable(), VulnerabilityMarks.XSS_MARK] | { - it[0] - } | { - it[1] - } + target | shouldTaint + string('string') | true + stringBuilder('stringBuilder') | true + date() | true + taintable() | true } - void 'test value source for #method'() { + void 'test taintIfTainted keeping ranges'() { given: - final span = Mock(AgentSpan) - tracer.activeSpan() >> span - final reqCtx = Mock(RequestContext) - span.getRequestContext() >> reqCtx - final ctx = new IastRequestContext() - reqCtx.getData(RequestContextSlot.IAST) >> ctx + def (target, input) = suite + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] - when: - module."$method"(source, name, value) + when: 'input is not tainted' + module.taintIfTainted(target, input, true, NOT_MARKED) then: - 1 * tracer.activeSpan() >> span - 1 * span.getRequestContext() >> reqCtx - 1 * reqCtx.getData(RequestContextSlot.IAST) >> ctx - 0 * _ - ctx.getTaintedObjects().get(name) == null - def to = ctx.getTaintedObjects().get(value) - to != null - to.get() == value - to.ranges.size() == 1 - to.ranges[0].start == 0 - to.ranges[0].length == value.length() - to.ranges[0].source == new Source(source, name, value) + assert getTaintedObject(target) == null + + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfTainted(target, input, true, NOT_MARKED) + + then: + final tainted = getTaintedObject(target) + if (target instanceof Taintable) { + // only first range is kept + assertTainted(tainted, [taintedFrom.ranges[0]] as Range[]) + } else { + assertTainted(tainted, taintedFrom.ranges) + } where: - method | name | value | source - 'taint' | null | 'value' | SourceTypes.REQUEST_PARAMETER_VALUE - 'taint' | 'name' | 'value' | SourceTypes.REQUEST_PARAMETER_VALUE + suite << taintIfSuite() } - void 'taint with context for #method'() { - setup: - def ctx = new IastRequestContext() + void 'test taintIfTainted keeping ranges with a mark'() { + given: + def (target, input) = suite + Assume.assumeFalse(target instanceof Taintable) // taintable does not support multiple ranges or marks + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] + final mark = VulnerabilityMarks.UNVALIDATED_REDIRECT_MARK - when: - module."$method"(ctx as Object, source, name, value) + when: 'input is not tainted' + module.taintIfTainted(target, input, true, mark) then: - ctx.getTaintedObjects().get(name) == null - def to = ctx.getTaintedObjects().get(value) - to != null - to.get() == value - to.ranges.size() == 1 - to.ranges[0].start == 0 - to.ranges[0].length == value.length() - to.ranges[0].source == new Source(source, name, value) - 0 * _ + assert getTaintedObject(target) == null + + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfTainted(target, input, true, mark) + + then: + final tainted = getTaintedObject(target) + assertTainted(tainted, taintedFrom.ranges, mark) where: - method | name | value | source - 'taint' | null | "value" | SourceTypes.REQUEST_PATH_PARAMETER - 'taint' | "" | "value" | SourceTypes.REQUEST_PATH_PARAMETER - 'taint' | "param" | "value" | SourceTypes.REQUEST_PATH_PARAMETER + suite << taintIfSuite() } - void 'test taintObject'() { - when: - module.taintObject(origin, toTaint) + void 'test taintIfTainted not keeping ranges'() { + given: + def (target, input) = suite + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] + + when: 'input is not tainted' + module.taintIfTainted(target, input, false, NOT_MARKED) then: - assertTainted(toTaint) + assert getTaintedObject(target) == null + + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfTainted(target, input, false, NOT_MARKED) + + then: + final tainted = getTaintedObject(target) + assertTainted(tainted, [highestPriorityRange(taintedFrom.ranges)] as Range[]) where: - origin | toTaint - SourceTypes.REQUEST_PARAMETER_VALUE | new Object() - SourceTypes.REQUEST_PARAMETER_VALUE | new MockTaintable() + suite << taintIfSuite() } - void 'test taintObjects[array]'() { - when: - module.taintObjects(origin, new Object[]{ - toTaint - }) + void 'test taintIfTainted not keeping ranges with a mark'() { + given: + def (target, input) = suite + Assume.assumeFalse(target instanceof Taintable) // taintable does not support marks + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] + final mark = VulnerabilityMarks.LDAP_INJECTION_MARK + + when: 'input is not tainted' + module.taintIfTainted(target, input, false, mark) then: - assertTainted(toTaint) + assert getTaintedObject(target) == null + + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfTainted(target, input, false, mark) + + then: + final tainted = getTaintedObject(target) + assertTainted(tainted, [highestPriorityRange(taintedFrom.ranges)] as Range[], mark) where: - origin | toTaint - SourceTypes.REQUEST_PARAMETER_VALUE | new Object() - SourceTypes.REQUEST_PARAMETER_VALUE | new MockTaintable() + suite << taintIfSuite() } - void 'onJsonFactoryCreateParser'() { + void 'test taintIfAnyTainted keeping ranges'() { given: - final taintedObjects = ctx.getTaintedObjects() - def shouldBeTainted = true + def (target, input) = suite + final inputs = ['test', input].toArray() + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] - def firstParam - if (param1 instanceof String) { - firstParam = addFromTaintFormat(taintedObjects, param1) - objectHolder.add(firstParam) - } else { - firstParam = param1 - } + when: 'input is not tainted' + module.taintIfAnyTainted(target, inputs, true, NOT_MARKED) - def secondParam - if (param2 instanceof String) { - secondParam = addFromTaintFormat(taintedObjects, param2) - objectHolder.add(secondParam) - shouldBeTainted = fromTaintFormat(param2) != null + then: + assert getTaintedObject(target) == null + + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfAnyTainted(target, inputs, true, NOT_MARKED) + + then: + final tainted = getTaintedObject(target) + if (target instanceof Taintable) { + // only first range is kept + assertTainted(tainted, [taintedFrom.ranges[0]] as Range[]) } else { - secondParam = param2 + assertTainted(tainted, taintedFrom.ranges) } - if (shouldBeTainted) { - def ranges = new Range[1] - ranges[0] = new Range(0, Integer.MAX_VALUE, new Source((byte) 1, "test", "test"), NOT_MARKED) - taintedObjects.taint(secondParam, ranges) - } + where: + suite << taintIfSuite() + } - when: - module.taintIfInputIsTainted(firstParam, secondParam) + void 'test taintIfAnyTainted keeping ranges with a mark'() { + given: + def (target, input) = suite + Assume.assumeFalse(target instanceof Taintable) // taintable does not support multiple ranges or marks + final inputs = ['test', input].toArray() + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] + final mark = VulnerabilityMarks.UNVALIDATED_REDIRECT_MARK + + when: 'input is not tainted' + module.taintIfAnyTainted(target, inputs, true, mark) then: - def to = ctx.getTaintedObjects().get(param1) - if (shouldBeTainted) { - assert to != null - assert to.get() == param1 - if (param1 instanceof String) { - final ranges = to.getRanges() - assert ranges.length == 1 - assert ranges[0].start == 0 - assert ranges[0].length == param1.length() - } else { - final ranges = to.getRanges() - assert ranges.length == 1 - assert ranges[0].start == 0 - assert ranges[0].length == Integer.MAX_VALUE - } - } else { - assert to == null - } + assert getTaintedObject(target) == null + + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfAnyTainted(target, inputs, true, mark) + + then: + final tainted = getTaintedObject(target) + assertTainted(tainted, taintedFrom.ranges, mark) where: - param1 | param2 - '123' | new Object() - new Object() | new Object() - new Object() | '123' - new Object() | '==>123<==' + suite << taintIfSuite() } - void 'test first tainted source'() { - when: - final before = module.firstTaintedSource(target) + void 'test taintIfAnyTainted not keeping ranges'() { + given: + def (target, input) = suite + final inputs = ['test', input].toArray() + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] + + when: 'input is not tainted' + module.taintIfAnyTainted(target, inputs, false, NOT_MARKED) then: - before == null + assert getTaintedObject(target) == null - when: - module.taintObject(origin, target) - final after = module.firstTaintedSource(target) + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfAnyTainted(target, inputs, false, NOT_MARKED) then: - after.origin == origin + final tainted = getTaintedObject(target) + assertTainted(tainted, [highestPriorityRange(taintedFrom.ranges)] as Range[]) where: - target | origin - 'this is a string' | SourceTypes.REQUEST_PARAMETER_VALUE - new Object() | SourceTypes.REQUEST_PARAMETER_VALUE - new MockTaintable() | SourceTypes.REQUEST_PARAMETER_VALUE + suite << taintIfSuite() } - void 'test taintIfInputIsTaintedWithMarks marks ranges for #mark'() { + void 'test taintIfAnyTainted not keeping ranges with a mark'() { given: - final toTaint = 'this is a string' - final tainted = new Object() - ctx.getTaintedObjects().taint(tainted, previousRanges) - objectHolder.add(toTaint) + def (target, input) = suite + Assume.assumeFalse(target instanceof Taintable) // taintable does not support marks + final inputs = ['test', input].toArray() + final source = taintedSource() + final ranges = [new Range(0, 1, source, NOT_MARKED), new Range(1, 1, source, NOT_MARKED)] as Range[] + final mark = VulnerabilityMarks.LDAP_INJECTION_MARK - when: - module.taintIfInputIsTaintedWithMarks(toTaint, tainted, mark) + when: 'input is not tainted' + module.taintIfAnyTainted(target, inputs, false, mark) then: - final to = ctx.getTaintedObjects().get(toTaint) - final ranges = to.getRanges() - ranges != null && ranges.length == 1 - ranges[0].marks == expected + assert getTaintedObject(target) == null + + when: 'input is tainted' + final taintedFrom = taintObject(input, ranges) + module.taintIfAnyTainted(target, inputs, false, mark) + + then: + final tainted = getTaintedObject(target) + assertTainted(tainted, [highestPriorityRange(taintedFrom.ranges)] as Range[], mark) where: - previousRanges | mark | expected - [new Range(0, Integer.MAX_VALUE, getDefaultSource(), NOT_MARKED)] as Range[] | XSS_MARK | XSS_MARK - [new Range(0, 1, getDefaultSource(), SQL_INJECTION_MARK), new Range(2, 3, getDefaultSource(), NOT_MARKED)] as Range[] | XSS_MARK | XSS_MARK - [new Range(2, 3, getDefaultSource(), NOT_MARKED), new Range(0, 1, getDefaultSource(), SQL_INJECTION_MARK)] as Range[] | XSS_MARK | XSS_MARK - [new Range(2, 3, getDefaultSource(), XPATH_INJECTION_MARK), new Range(0, 1, getDefaultSource(), SQL_INJECTION_MARK)] as Range[] | XSS_MARK | getMarks(XPATH_INJECTION_MARK, XSS_MARK) + suite << taintIfSuite() } void 'test taint deeply'() { @@ -479,81 +356,222 @@ class PropagationModuleTest extends IastModuleImplTestBase { final target = [Hello: " World!", Age: 25] when: - module.taintDeeply(null, SourceTypes.GRPC_BODY, target) + module.taintDeeply(target, SourceTypes.GRPC_BODY, { true }) then: - ctx.getTaintedObjects().empty + final taintedObjects = ctx.taintedObjects + target.keySet().each { key -> + assert taintedObjects.get(key) != null + } + assert taintedObjects.get(target['Hello']) != null + assert taintedObjects.size() == 3 // two keys and one string value + } + + void 'test taint deeply char sequence'() { + given: + final target = stringBuilder('taint me') when: - module.taintDeeply(ctx, SourceTypes.GRPC_BODY, target) + module.taintDeeply(target, SourceTypes.GRPC_BODY, { true }) then: final taintedObjects = ctx.taintedObjects - target.keySet().each { - key -> - assert taintedObjects.get(key) != null + assert taintedObjects.size() == 1 + final tainted = taintedObjects.get(target) + assert tainted != null + final source = tainted.ranges[0].source + assert source.origin == SourceTypes.GRPC_BODY + assert source.value == target.toString() + } + + void 'test is tainted and find source'() { + given: + if (source != null) { + taintObject(target, source) } - assert taintedObjects.get(target['Hello']) != null - assert taintedObjects.size() == 3 // two keys and one string value + + when: + final tainted = module.isTainted(target) + + then: + tainted == (source != null) + + when: + final foundSource = module.findSource(target) + + then: + foundSource == source + + where: + target | source + string('string') | null + stringBuilder('stringBuilder') | null + date() | null + taintable() | null + string('string') | taintedSource() + stringBuilder('stringBuilder') | taintedSource() + date() | taintedSource() + taintable() | taintedSource() + } + + void 'test source names over threshold'() { + given: + final maxSize = Config.get().iastTruncationMaxValueLength + + when: + module.taint(target, SourceTypes.REQUEST_PARAMETER_VALUE) + + then: + final tainted = ctx.getTaintedObjects().get(target) + tainted != null + final sourceValue = tainted.ranges.first().source.value + sourceValue.length() <= target.length() + sourceValue.length() <= maxSize + + where: + target | _ + string((0..Config.get().getIastTruncationMaxValueLength() * 2).join('')) | _ + stringBuilder((0..Config.get().getIastTruncationMaxValueLength() * 2).join('')) | _ } - private E taint(final E toTaint) { - final source = new Source(SourceTypes.REQUEST_PARAMETER_VALUE, null, null) - if (toTaint instanceof Taintable) { - toTaint.$$DD$setSource(source) + void 'test that source names should not make a strong reference over the value'() { + given: + final name = 'name' + + when: + module.taint(name, SourceTypes.REQUEST_PARAMETER_NAME, name) + + then: + final tainted = ctx.getTaintedObjects().get(name) + final taintedName = tainted.ranges[0].source.name + assert !taintedName.is(name) : 'Weak value should not be retained by the source name' + } + + private List> taintIfSuite() { + return [ + Tuple.tuple(string('string'), string('string')), + Tuple.tuple(string('string'), stringBuilder('stringBuilder')), + Tuple.tuple(string('string'), date()), + Tuple.tuple(string('string'), taintable()), + Tuple.tuple(stringBuilder('stringBuilder'), string('string')), + Tuple.tuple(stringBuilder('stringBuilder'), stringBuilder('stringBuilder')), + Tuple.tuple(stringBuilder('stringBuilder'), date()), + Tuple.tuple(stringBuilder('stringBuilder'), taintable()), + Tuple.tuple(date(), string('string')), + Tuple.tuple(date(), stringBuilder('stringBuilder')), + Tuple.tuple(date(), date()), + Tuple.tuple(date(), taintable()), + Tuple.tuple(taintable(), string('string')), + Tuple.tuple(taintable(), stringBuilder('stringBuilder')), + Tuple.tuple(taintable(), date()), + Tuple.tuple(taintable(), taintable()) + ] + } + + private TaintedObject getTaintedObject(final Object target) { + if (target instanceof Taintable) { + final source = (target as Taintable).$$DD$getSource() as Source + return source == null ? null : new TaintedObject(target, Ranges.forObject(source), null) + } + return ctx.getTaintedObjects().get(target) + } + + private TaintedObject taintObject(final Object target, Source source, int mark = NOT_MARKED) { + if (target instanceof Taintable) { + target.$$DD$setSource(source) + } else if (target instanceof CharSequence) { + ctx.getTaintedObjects().taint(target, Ranges.forCharSequence(target, source, mark)) } else { - ctx.taintedObjects.taintInputObject(toTaint, source) - objectHolder.add(toTaint) + ctx.getTaintedObjects().taint(target, Ranges.forObject(source, mark)) } - return toTaint + return getTaintedObject(target) } - private void assertTainted(final Object toTaint) { - final tainted = ctx.getTaintedObjects().get(toTaint) - if (toTaint instanceof Taintable) { - assert tainted == null - assert toTaint.$$DD$getSource() != null + private TaintedObject taintObject(final Object target, Range[] ranges) { + if (target instanceof Taintable) { + target.$$DD$setSource(ranges[0].getSource()) } else { - assert tainted != null + ctx.getTaintedObjects().taint(target, ranges) } + return getTaintedObject(target) } - private void assertNotTainted(final Object toTaint) { - final tainted = ctx.getTaintedObjects().get(toTaint) - assert tainted == null - if (toTaint instanceof Taintable) { - assert toTaint.$$DD$getSource() == null + private String string(String value, Source source = null, int mark = NOT_MARKED) { + final result = new String(value) + if (source != null) { + taintObject(result, source, mark) } + return result } - private static Source getDefaultSource() { - return new Source(SourceTypes.REQUEST_PARAMETER_VALUE, null, null) + private StringBuilder stringBuilder(String value, Source source = null, int mark = NOT_MARKED) { + final result = new StringBuilder(value) + if (source != null) { + taintObject(result, source, mark) + } + return result + } + + private Date date(Source source = null, int mark = NOT_MARKED) { + final result = new Date() + if (source != null) { + taintObject(result, source, mark) + } + return result } - private static int getMarks(int ... marks) { - int result = NOT_MARKED - for (int mark : marks) { - result = result | mark + private Taintable taintable(Source source = null) { + final result = new MockTaintable() + if (source != null) { + taintObject(result, source) } return result } - /** - * Mocking makes the test a bit more confusing*/ - private static final class MockTaintable implements Taintable { + private Source taintedSource(String value = 'value') { + return new Source(SourceTypes.REQUEST_PARAMETER_VALUE, 'name', value) + } + + private static void assertTainted(final TaintedObject tainted, final Range[] ranges, final int mark = NOT_MARKED) { + assert tainted != null + final originalValue = tainted.get() + assert tainted.ranges.length == ranges.length + ranges.eachWithIndex { Range expected, int i -> + final range = tainted.ranges[i] + if (mark == NOT_MARKED) { + assert range.marks == expected.marks + } else { + assert (range.marks & mark) > 0 + } + final source = range.source + assert !source.name.is(originalValue): 'Weak value should not be retained by the source name' + assert !source.value.is(originalValue): 'Weak value should not be retained by the source value' + + final expectedSource = expected.source + assert source.origin == expectedSource.origin + assert source.name == expectedSource.name + assert source.value == expectedSource.value + } + } + private static class MockTaintable implements Taintable { private Source source - @Override @SuppressWarnings('CodeNarc') + @Override Source $$DD$getSource() { return source } - @Override @SuppressWarnings('CodeNarc') + @Override void $$DD$setSource(Source source) { this.source = source } + + @Override + String toString() { + return Taintable.name + } } } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/CommandInjectionModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/CommandInjectionModuleTest.groovy index 9bc9148bb30..30b8adf5166 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/CommandInjectionModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/CommandInjectionModuleTest.groovy @@ -11,7 +11,7 @@ import datadog.trace.api.iast.sink.CommandInjectionModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan import groovy.transform.CompileDynamic -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat import static com.datadog.iast.taint.TaintUtils.taintFormat @@ -27,7 +27,7 @@ class CommandInjectionModuleTest extends IastModuleImplTestBase { private AgentSpan span def setup() { - module = registerDependencies(new CommandInjectionModuleImpl()) + module = new CommandInjectionModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HstsMissingHeaderModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HstsMissingHeaderModuleTest.groovy index 58009e2d5c7..505776b3ea4 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HstsMissingHeaderModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HstsMissingHeaderModuleTest.groovy @@ -1,13 +1,10 @@ package com.datadog.iast.sink -import com.datadog.iast.HasDependencies import com.datadog.iast.IastModuleImplTestBase import com.datadog.iast.IastRequestContext import com.datadog.iast.RequestEndedHandler import com.datadog.iast.model.Vulnerability import com.datadog.iast.model.VulnerabilityType -import com.datadog.iast.overhead.OverheadController -import datadog.trace.api.Config import datadog.trace.api.gateway.Flow import datadog.trace.api.gateway.IGSpanInfo import datadog.trace.api.gateway.RequestContext @@ -15,9 +12,8 @@ import datadog.trace.api.gateway.RequestContextSlot import datadog.trace.api.iast.InstrumentationBridge import datadog.trace.api.internal.TraceSegment import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import datadog.trace.util.stacktrace.StackWalker -public class HstsMissingHeaderModuleTest extends IastModuleImplTestBase { +class HstsMissingHeaderModuleTest extends IastModuleImplTestBase { private List objectHolder @@ -29,7 +25,7 @@ public class HstsMissingHeaderModuleTest extends IastModuleImplTestBase { def setup() { InstrumentationBridge.clearIastModules() - module = registerDependencies(new HstsMissingHeaderModuleImpl()) + module = new HstsMissingHeaderModuleImpl(dependencies) InstrumentationBridge.registerIastModule(module) objectHolder = [] ctx = new IastRequestContext() @@ -46,14 +42,9 @@ public class HstsMissingHeaderModuleTest extends IastModuleImplTestBase { void 'hsts vulnerability'() { given: Vulnerability savedVul1 - final OverheadController overheadController = Mock(OverheadController) final iastCtx = Mock(IastRequestContext) iastCtx.getxForwardedProto() >> 'https' iastCtx.getContentType() >> "text/html" - final StackWalker stackWalker = Mock(StackWalker) - final dependencies = new HasDependencies.Dependencies( - Config.get(), reporter, overheadController, stackWalker - ) final handler = new RequestEndedHandler(dependencies) final TraceSegment traceSegment = Mock(TraceSegment) final reqCtx = Mock(RequestContext) @@ -96,15 +87,9 @@ public class HstsMissingHeaderModuleTest extends IastModuleImplTestBase { void 'no hsts vulnerability reported'() { given: - Vulnerability savedVul1 - final OverheadController overheadController = Mock(OverheadController) final iastCtx = Mock(IastRequestContext) iastCtx.getxForwardedProto() >> 'https' iastCtx.getContentType() >> "text/html" - final StackWalker stackWalker = Mock(StackWalker) - final dependencies = new HasDependencies.Dependencies( - Config.get(), reporter, overheadController, stackWalker - ) final handler = new RequestEndedHandler(dependencies) final TraceSegment traceSegment = Mock(TraceSegment) final reqCtx = Mock(RequestContext) diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HttpResponseHeaderModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HttpResponseHeaderModuleTest.groovy index f884b16f60d..f3e30e1e180 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HttpResponseHeaderModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/HttpResponseHeaderModuleTest.groovy @@ -25,13 +25,13 @@ class HttpResponseHeaderModuleTest extends IastModuleImplTestBase { def setup() { InstrumentationBridge.clearIastModules() - module = registerDependencies(new HttpResponseHeaderModuleImpl()) + module = new HttpResponseHeaderModuleImpl(dependencies) InstrumentationBridge.registerIastModule(module) InstrumentationBridge.registerIastModule(new InsecureCookieModuleImpl()) InstrumentationBridge.registerIastModule(new NoHttpOnlyCookieModuleImpl()) InstrumentationBridge.registerIastModule(new NoSameSiteCookieModuleImpl()) - InstrumentationBridge.registerIastModule(new HstsMissingHeaderModuleImpl()) - InstrumentationBridge.registerIastModule(new UnvalidatedRedirectModuleImpl()) + InstrumentationBridge.registerIastModule(new HstsMissingHeaderModuleImpl(dependencies)) + InstrumentationBridge.registerIastModule(new UnvalidatedRedirectModuleImpl(dependencies)) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/InsecureCookieModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/InsecureCookieModuleTest.groovy index bf74a709519..b160f37a28b 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/InsecureCookieModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/InsecureCookieModuleTest.groovy @@ -24,7 +24,7 @@ class InsecureCookieModuleTest extends IastModuleImplTestBase { def setup() { InstrumentationBridge.clearIastModules() - module = registerDependencies(new HttpResponseHeaderModuleImpl()) + module = new HttpResponseHeaderModuleImpl(dependencies) InstrumentationBridge.registerIastModule(new InsecureCookieModuleImpl()) objectHolder = [] ctx = new IastRequestContext() diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/LdapInjectionModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/LdapInjectionModuleTest.groovy index 2159276a958..682e8dfef39 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/LdapInjectionModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/LdapInjectionModuleTest.groovy @@ -11,7 +11,7 @@ import datadog.trace.api.iast.sink.LdapInjectionModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan import groovy.transform.CompileDynamic -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat import static com.datadog.iast.taint.TaintUtils.taintFormat @@ -27,7 +27,7 @@ class LdapInjectionModuleTest extends IastModuleImplTestBase { private AgentSpan span def setup() { - module = registerDependencies(new LdapInjectionModuleImpl()) + module = new LdapInjectionModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoHttpCookieModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoHttpCookieModuleTest.groovy index 28fda1aeecb..a246f30ee9f 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoHttpCookieModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoHttpCookieModuleTest.groovy @@ -24,7 +24,7 @@ class NoHttpCookieModuleTest extends IastModuleImplTestBase { def setup() { InstrumentationBridge.clearIastModules() - module = registerDependencies(new HttpResponseHeaderModuleImpl()) + module = new HttpResponseHeaderModuleImpl(dependencies) InstrumentationBridge.registerIastModule(new NoHttpOnlyCookieModuleImpl()) objectHolder = [] ctx = new IastRequestContext() diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoSameSiteCookieModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoSameSiteCookieModuleTest.groovy index f4a97aaffa1..4e32a9b7722 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoSameSiteCookieModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/NoSameSiteCookieModuleTest.groovy @@ -24,7 +24,7 @@ class NoSameSiteCookieModuleTest extends IastModuleImplTestBase { def setup() { InstrumentationBridge.clearIastModules() - module = registerDependencies(new HttpResponseHeaderModuleImpl()) + module = new HttpResponseHeaderModuleImpl(dependencies) InstrumentationBridge.registerIastModule(new NoSameSiteCookieModuleImpl()) objectHolder = [] ctx = new IastRequestContext() diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/PathTraversalModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/PathTraversalModuleTest.groovy index d55e2975e7e..6c25122e393 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/PathTraversalModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/PathTraversalModuleTest.groovy @@ -10,7 +10,7 @@ import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.sink.PathTraversalModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat import static com.datadog.iast.taint.TaintUtils.fromTaintFormat import static com.datadog.iast.taint.TaintUtils.getStringFromTaintFormat @@ -27,7 +27,7 @@ class PathTraversalModuleTest extends IastModuleImplTestBase { private IastRequestContext ctx def setup() { - module = registerDependencies(new PathTraversalModuleImpl()) + module = new PathTraversalModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SqlInjectionModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SqlInjectionModuleTest.groovy index 3041c928a2a..69b30fe5ef3 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SqlInjectionModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SqlInjectionModuleTest.groovy @@ -10,7 +10,7 @@ import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.sink.SqlInjectionModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat import static com.datadog.iast.taint.TaintUtils.taintFormat import static datadog.trace.api.iast.sink.SqlInjectionModule.DATABASE_PARAMETER @@ -24,7 +24,7 @@ class SqlInjectionModuleTest extends IastModuleImplTestBase { private IastRequestContext ctx def setup() { - module = registerDependencies(new SqlInjectionModuleImpl()) + module = new SqlInjectionModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SsrfModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SsrfModuleTest.groovy index 6884da0e616..c379cea4b8f 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SsrfModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/SsrfModuleTest.groovy @@ -6,6 +6,7 @@ import com.datadog.iast.model.Range import com.datadog.iast.model.Source import com.datadog.iast.model.Vulnerability import com.datadog.iast.model.VulnerabilityType +import com.datadog.iast.taint.Ranges import datadog.trace.api.gateway.RequestContext import datadog.trace.api.gateway.RequestContextSlot import datadog.trace.api.iast.SourceTypes @@ -24,7 +25,7 @@ class SsrfModuleTest extends IastModuleImplTestBase { private AgentSpan span def setup() { - module = registerDependencies(new SsrfModuleImpl()) + module = new SsrfModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { @@ -82,6 +83,6 @@ class SsrfModuleTest extends IastModuleImplTestBase { } private void taint(final Object value) { - ctx.getTaintedObjects().taintInputObject(value, new Source(SourceTypes.REQUEST_PARAMETER_VALUE, 'name', value.toString())) + ctx.getTaintedObjects().taint(value, Ranges.forObject(new Source(SourceTypes.REQUEST_PARAMETER_VALUE, 'name', value.toString()))) } } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/StacktraceLeakModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/StacktraceLeakModuleTest.groovy new file mode 100644 index 00000000000..ecf8aaa3bd0 --- /dev/null +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/StacktraceLeakModuleTest.groovy @@ -0,0 +1,51 @@ +package com.datadog.iast.sink + +import com.datadog.iast.IastModuleImplTestBase +import com.datadog.iast.model.Evidence +import com.datadog.iast.model.Vulnerability +import com.datadog.iast.model.VulnerabilityType +import datadog.trace.api.iast.sink.StacktraceLeakModule +import datadog.trace.bootstrap.instrumentation.api.AgentSpan + +class StacktraceLeakModuleTest extends IastModuleImplTestBase { + private StacktraceLeakModule module + + def setup() { + module = new StacktraceLeakModuleImpl(dependencies) + } + + void 'iast stacktrace leak module'() { + given: + final spanId = 123456 + final span = Mock(AgentSpan) + + def throwable = new Exception('some exception') + def moduleName = 'moduleName' + def className = 'className' + def methodName = 'methodName' + + when: + module.onStacktraceLeak(throwable, moduleName, className, methodName) + + then: + 1 * tracer.activeSpan() >> span + 1 * span.getSpanId() >> spanId + 1 * span.getServiceName() + 1 * reporter.report(_, _) >> { args -> + Vulnerability vuln = args[1] as Vulnerability + assert vuln != null + assert vuln.getType() == VulnerabilityType.STACKTRACE_LEAK + assert vuln.getEvidence() == new Evidence('ExceptionHandler in moduleName \r\nthrown java.lang.Exception') + assert vuln.getLocation() != null + } + 0 * _ + } + + void 'iast stacktrace leak no exception'() { + when: + module.onStacktraceLeak(null, null, null, null) + + then: + 0 * _ + } +} diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/TrustBoundaryViolationModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/TrustBoundaryViolationModuleTest.groovy index e50a7b26674..a3fce16d195 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/TrustBoundaryViolationModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/TrustBoundaryViolationModuleTest.groovy @@ -5,6 +5,7 @@ import com.datadog.iast.IastRequestContext import com.datadog.iast.model.Source import com.datadog.iast.model.Vulnerability import com.datadog.iast.model.VulnerabilityType +import com.datadog.iast.taint.Ranges import datadog.trace.api.gateway.RequestContext import datadog.trace.api.gateway.RequestContextSlot import datadog.trace.api.iast.InstrumentationBridge @@ -24,7 +25,7 @@ class TrustBoundaryViolationModuleTest extends IastModuleImplTestBase { def setup() { InstrumentationBridge.clearIastModules() - module = registerDependencies(new TrustBoundaryViolationModuleImpl()) + module = new TrustBoundaryViolationModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { @@ -60,7 +61,7 @@ class TrustBoundaryViolationModuleTest extends IastModuleImplTestBase { given: Vulnerability savedVul final name = "name" - ctx.getTaintedObjects().taintInputString(name, new Source(SourceTypes.NONE, null, null)) + ctx.getTaintedObjects().taint(name, Ranges.forCharSequence(name, new Source(SourceTypes.NONE, null, null))) when: module.onSessionValue(name, "value") @@ -77,7 +78,7 @@ class TrustBoundaryViolationModuleTest extends IastModuleImplTestBase { Vulnerability savedVul final name = "name" final badValue = "theValue" - ctx.getTaintedObjects().taintInputString(badValue, new Source(SourceTypes.NONE, null, null)) + ctx.getTaintedObjects().taint(badValue, Ranges.forCharSequence(badValue, new Source(SourceTypes.NONE, null, null))) when: module.onSessionValue(name, badValue) @@ -95,7 +96,7 @@ class TrustBoundaryViolationModuleTest extends IastModuleImplTestBase { Vulnerability savedVul final name = "name" final badValue = "badValue" - ctx.getTaintedObjects().taintInputString(badValue, new Source(SourceTypes.NONE, null, null)) + ctx.getTaintedObjects().taint(badValue, Ranges.forCharSequence(badValue, new Source(SourceTypes.NONE, null, null))) final values = ["A", "B", badValue] when: @@ -113,7 +114,7 @@ class TrustBoundaryViolationModuleTest extends IastModuleImplTestBase { Vulnerability savedVul final name = "name" final badValue = "badValue" - ctx.getTaintedObjects().taintInputString(badValue, new Source(SourceTypes.NONE, null, null)) + ctx.getTaintedObjects().taint(badValue, Ranges.forCharSequence(badValue, new Source(SourceTypes.NONE, null, null))) final values = new String[3] values[0] = "A" values[1] = "B" @@ -134,7 +135,7 @@ class TrustBoundaryViolationModuleTest extends IastModuleImplTestBase { Vulnerability savedVul final name = "name" final badValue = "badValue" - ctx.getTaintedObjects().taintInputString(badValue, new Source(SourceTypes.NONE, null, null)) + ctx.getTaintedObjects().taint(badValue, Ranges.forCharSequence(badValue, new Source(SourceTypes.NONE, null, null))) final values = new LinkedHashMap() values.put("A", "A") values.put("B", "B") @@ -155,7 +156,7 @@ class TrustBoundaryViolationModuleTest extends IastModuleImplTestBase { Vulnerability savedVul final name = "name" final badValue = "badValue" - ctx.getTaintedObjects().taintInputString(badValue, new Source(SourceTypes.NONE, null, null)) + ctx.getTaintedObjects().taint(badValue, Ranges.forCharSequence(badValue, new Source(SourceTypes.NONE, null, null))) final value = new VisitableClass(name: badValue) when: diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/UnvalidatedRedirectModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/UnvalidatedRedirectModuleTest.groovy index 708b665a902..5dabb12cc52 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/UnvalidatedRedirectModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/UnvalidatedRedirectModuleTest.groovy @@ -6,6 +6,7 @@ import com.datadog.iast.model.Range import com.datadog.iast.model.Source import com.datadog.iast.model.Vulnerability import com.datadog.iast.model.VulnerabilityType +import com.datadog.iast.taint.Ranges import datadog.trace.api.gateway.RequestContext import datadog.trace.api.gateway.RequestContextSlot import datadog.trace.api.iast.InstrumentationBridge @@ -14,7 +15,7 @@ import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.sink.UnvalidatedRedirectModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat import static com.datadog.iast.taint.TaintUtils.taintFormat @@ -27,7 +28,7 @@ class UnvalidatedRedirectModuleTest extends IastModuleImplTestBase { private IastRequestContext ctx def setup() { - module = registerDependencies(new UnvalidatedRedirectModuleImpl()) + module = new UnvalidatedRedirectModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { @@ -64,7 +65,7 @@ class UnvalidatedRedirectModuleTest extends IastModuleImplTestBase { void 'iast module detects URI redirect (#value)'(final URI value, final String expected) { setup: - ctx.taintedObjects.taintInputObject(value, new Source(SourceTypes.NONE, null, null)) + ctx.taintedObjects.taint(value, Ranges.forObject(new Source(SourceTypes.NONE, null, null))) when: module.onURIRedirect(value) @@ -107,7 +108,7 @@ class UnvalidatedRedirectModuleTest extends IastModuleImplTestBase { void 'if onHeader receives a Location header call onRedirect'() { setup: - final urm = Spy(UnvalidatedRedirectModuleImpl) + final urm = Spy(new UnvalidatedRedirectModuleImpl(dependencies)) InstrumentationBridge.registerIastModule(urm) when: diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakCipherModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakCipherModuleTest.groovy index 5b71caa24d4..802df43ec28 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakCipherModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakCipherModuleTest.groovy @@ -12,7 +12,7 @@ class WeakCipherModuleTest extends IastModuleImplTestBase { private WeakCipherModule module def setup() { - module = registerDependencies(new WeakCipherModuleImpl()) + module = new WeakCipherModuleImpl(dependencies) } void 'iast module vulnerable cipher algorithm'(final String algorithm){ diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakHashModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakHashModuleTest.groovy index b3e83664558..2cfebb744e0 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakHashModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakHashModuleTest.groovy @@ -12,7 +12,7 @@ class WeakHashModuleTest extends IastModuleImplTestBase { private WeakHashModule module def setup() { - module = registerDependencies(new WeakHashModuleImpl()) + module = new WeakHashModuleImpl(dependencies) } void 'iast module vulnerable hash algorithm'(final String algorithm){ diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakRandomnessModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakRandomnessModuleTest.groovy index a7401b01a5f..fa3db3c8f19 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakRandomnessModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/WeakRandomnessModuleTest.groovy @@ -14,7 +14,7 @@ class WeakRandomnessModuleTest extends IastModuleImplTestBase { private AgentSpan span def setup() { - module = registerDependencies(new WeakRandomnessModuleImpl()) + module = new WeakRandomnessModuleImpl(dependencies) span = Mock(AgentSpan) { getSpanId() >> 123456 } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XContentTypeOptionsModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XContentTypeOptionsModuleTest.groovy index dfda4e18cdb..a8d377e968b 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XContentTypeOptionsModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XContentTypeOptionsModuleTest.groovy @@ -1,13 +1,10 @@ package com.datadog.iast.sink -import com.datadog.iast.HasDependencies import com.datadog.iast.IastModuleImplTestBase import com.datadog.iast.IastRequestContext import com.datadog.iast.RequestEndedHandler import com.datadog.iast.model.Vulnerability import com.datadog.iast.model.VulnerabilityType -import com.datadog.iast.overhead.OverheadController -import datadog.trace.api.Config import datadog.trace.api.gateway.Flow import datadog.trace.api.gateway.IGSpanInfo import datadog.trace.api.gateway.RequestContext @@ -15,7 +12,6 @@ import datadog.trace.api.gateway.RequestContextSlot import datadog.trace.api.iast.InstrumentationBridge import datadog.trace.api.internal.TraceSegment import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import datadog.trace.util.stacktrace.StackWalker public class XContentTypeOptionsModuleTest extends IastModuleImplTestBase { @@ -29,7 +25,7 @@ public class XContentTypeOptionsModuleTest extends IastModuleImplTestBase { def setup() { InstrumentationBridge.clearIastModules() - module = registerDependencies(new XContentTypeModuleImpl()) + module = new XContentTypeModuleImpl(dependencies) InstrumentationBridge.registerIastModule(module) objectHolder = [] ctx = new IastRequestContext() @@ -46,13 +42,8 @@ public class XContentTypeOptionsModuleTest extends IastModuleImplTestBase { void 'x content options sniffing vulnerability'() { given: Vulnerability savedVul1 - final OverheadController overheadController = Mock(OverheadController) final iastCtx = Mock(IastRequestContext) iastCtx.getContentType() >> "text/html" - final StackWalker stackWalker = Mock(StackWalker) - final dependencies = new HasDependencies.Dependencies( - Config.get(), reporter, overheadController, stackWalker - ) final handler = new RequestEndedHandler(dependencies) final TraceSegment traceSegment = Mock(TraceSegment) final reqCtx = Mock(RequestContext) @@ -93,15 +84,9 @@ public class XContentTypeOptionsModuleTest extends IastModuleImplTestBase { void 'no x content options sniffing reported'() { given: - Vulnerability savedVul1 - final OverheadController overheadController = Mock(OverheadController) final iastCtx = Mock(IastRequestContext) iastCtx.getxForwardedProto() >> 'https' iastCtx.getContentType() >> "text/html" - final StackWalker stackWalker = Mock(StackWalker) - final dependencies = new HasDependencies.Dependencies( - Config.get(), reporter, overheadController, stackWalker - ) final handler = new RequestEndedHandler(dependencies) final TraceSegment traceSegment = Mock(TraceSegment) final reqCtx = Mock(RequestContext) diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XPathInjectionModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XPathInjectionModuleTest.groovy index 6a44afd10b8..3638b6f431b 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XPathInjectionModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XPathInjectionModuleTest.groovy @@ -10,7 +10,7 @@ import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.sink.XPathInjectionModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat import static com.datadog.iast.taint.TaintUtils.taintFormat @@ -23,7 +23,7 @@ class XPathInjectionModuleTest extends IastModuleImplTestBase { private IastRequestContext ctx def setup() { - module = registerDependencies(new XPathInjectionModuleImpl()) + module = new XPathInjectionModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XssModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XssModuleTest.groovy index ff0340a99df..bbebbceae97 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XssModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/sink/XssModuleTest.groovy @@ -5,6 +5,7 @@ import com.datadog.iast.IastRequestContext import com.datadog.iast.model.Source import com.datadog.iast.model.Vulnerability import com.datadog.iast.model.VulnerabilityType +import com.datadog.iast.taint.Ranges import datadog.trace.api.gateway.RequestContext import datadog.trace.api.gateway.RequestContextSlot import datadog.trace.api.iast.SourceTypes @@ -12,7 +13,7 @@ import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.sink.XssModule import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.TaintUtils.addFromTaintFormat import static com.datadog.iast.taint.TaintUtils.taintFormat @@ -25,7 +26,7 @@ class XssModuleTest extends IastModuleImplTestBase { private IastRequestContext ctx def setup() { - module = registerDependencies(new XssModuleImpl()) + module = new XssModuleImpl(dependencies) objectHolder = [] ctx = new IastRequestContext() final reqCtx = Mock(RequestContext) { @@ -65,7 +66,7 @@ class XssModuleTest extends IastModuleImplTestBase { void 'module detects char[] XSS'() { setup: if (tainted) { - ctx.taintedObjects.taintInputObject(buf, new Source(SourceTypes.NONE, '', ''), mark) + ctx.taintedObjects.taint(buf, Ranges.forObject(new Source(SourceTypes.NONE, '', ''), mark)) } when: diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/source/WebModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/source/WebModuleTest.groovy deleted file mode 100644 index 1e93f5e3fb4..00000000000 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/source/WebModuleTest.groovy +++ /dev/null @@ -1,130 +0,0 @@ -package com.datadog.iast.source - -import com.datadog.iast.IastModuleImplTestBase -import com.datadog.iast.IastRequestContext -import com.datadog.iast.model.Source -import datadog.trace.api.gateway.RequestContext -import datadog.trace.api.gateway.RequestContextSlot -import datadog.trace.api.iast.SourceTypes -import datadog.trace.api.iast.source.WebModule -import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import groovy.transform.CompileDynamic - -@CompileDynamic -class WebModuleTest extends IastModuleImplTestBase { - - private WebModule module - - def setup() { - module = new WebModuleImpl() - } - - void 'test #method: null or empty'() { - when: - module."$method"(*args) - - then: - 0 * _ - - where: - method | args - 'onParameterNames' | [null] - 'onParameterNames' | [[]] - 'onParameterValues' | [null, null] - 'onParameterValues' | ['', []] - 'onParameterValues' | [null, null as String[]] - 'onParameterValues' | ['', [] as String[]] - 'onParameterValues' | [[:]] - 'onHeaderNames' | [null] - 'onHeaderNames' | [[]] - 'onHeaderValues' | [null, null] - 'onHeaderValues' | ['', []] - 'onCookieNames' | [null] - 'onCookieNames' | [[]] - } - - void 'test #method: without span'() { - when: - module."$method"(*args) - - then: - 1 * tracer.activeSpan() >> null - 0 * _ - - where: - method | args - 'onParameterNames' | [['param']] - 'onParameterValues' | ['name', ['value']] - 'onParameterValues' | ['name', ['value'] as String[]] - 'onParameterValues' | [[name: ['value'] as String[]]] - 'onHeaderNames' | [['header']] - 'onHeaderValues' | ['name', ['value']] - 'onCookieNames' | [['name']] - 'onNamed' | ['name', ['v1'], (byte)0] - 'onNamed' | ['name', ['v1'] as String[], (byte)0] - 'onNamed' | [[name: 'v1'], (byte)0] - } - - void 'onNamed #variant'() { - given: - final span = Mock(AgentSpan) - tracer.activeSpan() >> span - final reqCtx = Mock(RequestContext) - span.getRequestContext() >> reqCtx - final ctx = new IastRequestContext() - reqCtx.getData(RequestContextSlot.IAST) >> ctx - - when: - module.onNamed(*args, SourceTypes.REQUEST_PARAMETER_NAME) - - then: - 1 * tracer.activeSpan() >> span - 1 * span.getRequestContext() >> reqCtx - 1 * reqCtx.getData(RequestContextSlot.IAST) >> ctx - 0 * _ - def tos = ctx.taintedObjects - def to = tos.get('foo') - to.ranges.size() == 1 - to.ranges[0].start == 0 - to.ranges[0].length == 3 - to.ranges[0].source == new Source(SourceTypes.REQUEST_PARAMETER_NAME, 'var', 'foo') - - where: - variant | args - 'collection' | ['var', ['foo']] - 'array' | ['var', ['foo'] as String[]] - 'map' | [[var: ['foo'] as String[]]] - } - - void 'test #method'() { - given: - final span = Mock(AgentSpan) - tracer.activeSpan() >> span - final reqCtx = Mock(RequestContext) - span.getRequestContext() >> reqCtx - final ctx = new IastRequestContext() - reqCtx.getData(RequestContextSlot.IAST) >> ctx - - when: - module."$method"([name]) - - then: - 1 * tracer.activeSpan() >> span - 1 * span.getRequestContext() >> reqCtx - 1 * reqCtx.getData(RequestContextSlot.IAST) >> ctx - 0 * _ - def to = ctx.getTaintedObjects().get(name) - to != null - to.get() == name - to.ranges.size() == 1 - to.ranges[0].start == 0 - to.ranges[0].length == name.length() - to.ranges[0].source == new Source(source, name, name) - - where: - method | name | source - 'onParameterNames' | 'param' | SourceTypes.REQUEST_PARAMETER_NAME - 'onHeaderNames' | 'param' | SourceTypes.REQUEST_HEADER_NAME - 'onCookieNames' | 'param' | SourceTypes.REQUEST_COOKIE_NAME - } -} diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/RangesTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/RangesTest.groovy index 8184a45abe8..60525afa7e2 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/RangesTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/RangesTest.groovy @@ -8,7 +8,7 @@ import datadog.trace.api.iast.SourceTypes import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.test.util.DDSpecification -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static com.datadog.iast.taint.Ranges.mergeRanges import static com.datadog.iast.taint.Ranges.rangesProviderFor import static datadog.trace.api.iast.SourceTypes.REQUEST_HEADER_NAME @@ -23,7 +23,7 @@ class RangesTest extends DDSpecification { final source = new Source(SourceTypes.NONE, null, null) when: - final result = Ranges.forString(s, source, VulnerabilityMarks.SQL_INJECTION_MARK) + final result = Ranges.forCharSequence(s, source, VulnerabilityMarks.SQL_INJECTION_MARK) then: result != null diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintUtils.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintUtils.groovy index 7db1a75c673..88b21e9406b 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintUtils.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintUtils.groovy @@ -4,7 +4,7 @@ import com.datadog.iast.model.Range import com.datadog.iast.model.Source import datadog.trace.api.iast.SourceTypes -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED class TaintUtils { @@ -76,7 +76,7 @@ class TaintUtils { if (value instanceof String) { return addFromTaintFormat(tos, value as String) } - tos.taintInputObject(value, new Source(SourceTypes.NONE, null, null)) + tos.taint(value, Ranges.forObject(new Source(SourceTypes.NONE, null, null))) return value } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectTest.groovy index cb0351bf0b0..d809f5b8b69 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectTest.groovy @@ -7,7 +7,7 @@ import com.datadog.iast.model.Range import java.lang.ref.ReferenceQueue -import static com.datadog.iast.model.Range.NOT_MARKED +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED import static datadog.trace.api.iast.SourceTypes.REQUEST_HEADER_NAME class TaintedObjectTest extends Specification { diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsLazyTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsLazyTest.groovy deleted file mode 100644 index 8ed664ff27b..00000000000 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsLazyTest.groovy +++ /dev/null @@ -1,69 +0,0 @@ -package com.datadog.iast.taint - -import com.datadog.iast.IastModuleImplTestBase -import com.datadog.iast.IastRequestContext -import com.datadog.iast.model.Source -import datadog.trace.api.gateway.RequestContext -import datadog.trace.api.gateway.RequestContextSlot -import datadog.trace.bootstrap.instrumentation.api.AgentSpan - -class TaintedObjectsLazyTest extends IastModuleImplTestBase { - - private TaintedObjects delegate = Mock(TaintedObjects) - private IastRequestContext iastCtx - private RequestContext reqCtx - private AgentSpan span - - void setup() { - delegate = Mock(TaintedObjects) - iastCtx = Mock(IastRequestContext) - iastCtx.getTaintedObjects() >> delegate - reqCtx = Mock(RequestContext) - reqCtx.getData(RequestContextSlot.IAST) >> iastCtx - span = Mock(AgentSpan) - span.getRequestContext() >> reqCtx - } - - void 'get non lazy instance'() { - when: - final to = TaintedObjects.activeTaintedObjects() - - then: - 1 * tracer.activeSpan() >> span - !(to instanceof TaintedObjects.LazyTaintedObjects) - } - - void 'get lazy objects instance'() { - when: - final to = TaintedObjects.activeTaintedObjects(true) - - then: - to instanceof TaintedObjects.LazyTaintedObjects - - when: - to.&"$method".call(args as Object[]) - - then: 'first time the active tainted objects if fetched' - 1 * delegate._ - 1 * tracer.activeSpan() >> span - - when: - to.&"$method".call(args as Object[]) - - then: 'the active tainted objets is already fetched' - 1 * delegate._ - 0 * _ - - where: - method | args - 'getEstimatedSize' | [] - 'isFlat' | [] - 'taintInputString' | ['', new Source((byte) 0, null, null)] - 'taintInputObject' | ['', new Source((byte) 0, null, null)] - 'taint' | ['', Ranges.EMPTY] - 'get' | [''] - 'release' | [] - 'count' | [] - 'iterator' | [] - } -} diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsLogTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsLogTest.groovy index 5ba53745d51..ea01efa3989 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsLogTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsLogTest.groovy @@ -3,7 +3,6 @@ package com.datadog.iast.taint import ch.qos.logback.classic.Level import ch.qos.logback.classic.Logger import com.datadog.iast.IastSystem -import com.datadog.iast.model.Range import com.datadog.iast.model.Source import datadog.trace.api.iast.SourceTypes import datadog.trace.test.util.DDSpecification @@ -35,7 +34,7 @@ class TaintedObjectsLogTest extends DDSpecification { final value = "A" when: - def tainted = taintedObjects.taintInputString(value, new Source(SourceTypes.NONE, null, null)) + def tainted = taintedObjects.taint(value, Ranges.forCharSequence(value, new Source(SourceTypes.NONE, null, null))) then: noExceptionThrown() @@ -54,9 +53,8 @@ class TaintedObjectsLogTest extends DDSpecification { IastSystem.DEBUG = true logger.level = Level.ALL TaintedObjects taintedObjects = TaintedObjects.acquire() - taintedObjects.taint('A', [new Range(0, 1, new Source(SourceTypes.NONE, null, null), Range.NOT_MARKED)] as Range[]) - taintedObjects.taintInputString('B', new Source(SourceTypes.REQUEST_PARAMETER_NAME, 'test', 'value')) - taintedObjects.taintInputObject(new Date(), new Source(SourceTypes.REQUEST_HEADER_VALUE, 'test', 'value')) + final obj = 'A' + taintedObjects.taint(obj, Ranges.forCharSequence(obj, new Source(SourceTypes.NONE, null, null))) when: taintedObjects.release() @@ -64,5 +62,21 @@ class TaintedObjectsLogTest extends DDSpecification { then: noExceptionThrown() } + + void "test TaintedObjects api calls"() { + given: + IastSystem.DEBUG = true + logger.level = Level.ALL + TaintedObjects taintedObjects = TaintedObjects.acquire() + final obj = 'A' + + when: + taintedObjects.taint(obj, Ranges.forCharSequence(obj, new Source(SourceTypes.NONE, null, null))) + + then: + taintedObjects.size() == 1 + taintedObjects.iterator().size() == 1 + !taintedObjects.flat + } } diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsNoOpTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsNoOpTest.groovy index 671eaf246b7..c8f1f91fe86 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsNoOpTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/taint/TaintedObjectsNoOpTest.groovy @@ -12,13 +12,12 @@ class TaintedObjectsNoOpTest extends Specification { final toTaint = 'test' when: - final taintedString = instance.taintInputString(toTaint, new Source(0 as byte, 'test', 'test')) - final taintedObject = instance.taintInputObject(toTaint, new Source(0 as byte, 'test', 'test')) + final tainted = instance.taint(toTaint, Ranges.forCharSequence(toTaint, new Source(0 as byte, 'test', 'test'))) then: - taintedString == null - taintedObject == null + tainted == null instance.get(toTaint) == null + instance.count() == 0 instance.estimatedSize == 0 instance.size() == 0 !instance.flat diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/AbstractTelemetryCallbackTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/AbstractTelemetryCallbackTest.groovy index aefaf6c46a1..92f03e910d2 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/AbstractTelemetryCallbackTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/AbstractTelemetryCallbackTest.groovy @@ -1,7 +1,7 @@ package com.datadog.iast.telemetry -import com.datadog.iast.HasDependencies.Dependencies +import com.datadog.iast.Dependencies import com.datadog.iast.IastModuleImplTestBase import datadog.trace.api.Config import datadog.trace.api.gateway.RequestContext diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/TelemetryRequestEndedHandlerTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/TelemetryRequestEndedHandlerTest.groovy index 24e2ef187b5..30a61959d98 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/TelemetryRequestEndedHandlerTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/TelemetryRequestEndedHandlerTest.groovy @@ -3,6 +3,7 @@ package com.datadog.iast.telemetry import com.datadog.iast.IastRequestContext import com.datadog.iast.RequestEndedHandler import com.datadog.iast.model.Source +import com.datadog.iast.taint.Ranges import com.datadog.iast.taint.TaintedObjects import com.datadog.iast.telemetry.taint.TaintedObjectsWithTelemetry import datadog.trace.api.gateway.RequestContextSlot @@ -41,7 +42,7 @@ class TelemetryRequestEndedHandlerTest extends AbstractTelemetryCallbackTest { final handler = new TelemetryRequestEndedHandler(delegate) final toTaint = 'hello' final source = new Source(SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value') - iastCtx.taintedObjects.taintInputString(toTaint, source) + iastCtx.taintedObjects.taint(toTaint, Ranges.forCharSequence(toTaint, source)) when: handler.apply(reqCtx, span) diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetryTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetryTest.groovy index bca55f07c11..c20ca32dace 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetryTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/telemetry/taint/TaintedObjectsWithTelemetryTest.groovy @@ -2,13 +2,11 @@ package com.datadog.iast.telemetry.taint import com.datadog.iast.IastRequestContext import com.datadog.iast.model.Range -import com.datadog.iast.model.Source import com.datadog.iast.taint.Ranges import com.datadog.iast.taint.TaintedObject import com.datadog.iast.taint.TaintedObjects import datadog.trace.api.gateway.RequestContext import datadog.trace.api.gateway.RequestContextSlot -import datadog.trace.api.iast.SourceTypes import datadog.trace.api.iast.telemetry.IastMetric import datadog.trace.api.iast.telemetry.IastMetricCollector import datadog.trace.api.iast.telemetry.Verbosity @@ -74,13 +72,11 @@ class TaintedObjectsWithTelemetryTest extends DDSpecification { final taintedObjects = TaintedObjectsWithTelemetry.build(verbosity, Mock(TaintedObjects)) when: - taintedObjects.taintInputString('test', new Source(SourceTypes.REQUEST_PARAMETER_VALUE, 'name', 'value')) - taintedObjects.taintInputObject(new Date(), new Source(SourceTypes.REQUEST_HEADER_VALUE, 'name', 'value')) taintedObjects.taint('test', new Range[0]) then: if (IastMetric.EXECUTED_TAINTED.isEnabled(verbosity)) { - 3 * mockCollector.addMetric(IastMetric.EXECUTED_TAINTED, _, 1) // two calls with one element + 1 * mockCollector.addMetric(IastMetric.EXECUTED_TAINTED, _, 1) } else { 0 * mockCollector.addMetric } diff --git a/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml b/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml index 485afe29f2c..cf92538ae54 100644 --- a/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml +++ b/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml @@ -247,6 +247,38 @@ suite: } ] } + - type: 'VULNERABILITIES' + description: 'Query with single quoted string literal and null source' + input: > + [ + { + "type": "SQL_INJECTION", + "evidence": { + "value": "select * from users where username = 'user'", + "ranges": [ + { "start" : 38, "length" : 4, "source": { "origin": "http.request.body" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.body" } + ], + "vulnerabilities": [ + { + "type": "SQL_INJECTION", + "evidence": { + "valueParts": [ + { "value": "select * from users where username = '" }, + { "redacted": true, "source": 0, "pattern": "****" }, + { "value": "'" } + ] + } + } + ] + } - type: 'VULNERABILITIES' description: '$1 query with double quoted string literal $2' parameters: @@ -1753,6 +1785,40 @@ suite: ] } + + - type: 'VULNERABILITIES' + description: 'Tainted range based redaction - with null source ' + input: > + [ + { + "type": "XSS", + "evidence": { + "value": "this could be a super long text, so we need to reduce it before send it to the backend. This redaction strategy applies to XSS vulnerability but can be extended to future ones", + "ranges": [ + { "start" : 123, "length" : 3, "source": { "origin": "http.request.body" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.body" } + ], + "vulnerabilities": [ + { + "type": "XSS", + "evidence": { + "valueParts": [ + { "redacted": true }, + { "source": 0, "value": "XSS" }, + { "redacted": true } + ] + } + } + ] + } + - type: 'VULNERABILITIES' description: 'Tainted range based redaction - multiple ranges' input: > diff --git a/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/IastRequestContextPreparationTrait.groovy b/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/IastRequestContextPreparationTrait.groovy index ffa956d5ded..6e0b849f6ca 100644 --- a/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/IastRequestContextPreparationTrait.groovy +++ b/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/IastRequestContextPreparationTrait.groovy @@ -3,16 +3,9 @@ package com.datadog.iast.test import com.datadog.iast.IastRequestContext import com.datadog.iast.IastSystem import com.datadog.iast.model.Range -import com.datadog.iast.model.Source import com.datadog.iast.taint.TaintedObject import com.datadog.iast.taint.TaintedObjects -import datadog.trace.api.gateway.CallbackProvider -import datadog.trace.api.gateway.EventType -import datadog.trace.api.gateway.Events -import datadog.trace.api.gateway.Flow -import datadog.trace.api.gateway.IGSpanInfo -import datadog.trace.api.gateway.RequestContext -import datadog.trace.api.gateway.RequestContextSlot +import datadog.trace.api.gateway.* import datadog.trace.api.iast.InstrumentationBridge import datadog.trace.bootstrap.instrumentation.api.AgentTracer import org.slf4j.Logger @@ -68,22 +61,12 @@ trait IastRequestContextPreparationTrait { List objects = Collections.synchronizedList([]) - TaintedObject taintInputString(String obj, Source source, int mark) { - objects << obj - this.delegate.taintInputString(obj, source, mark) - logTaint obj - } - - TaintedObject taintInputObject(Object obj, Source source, int mark) { - objects << obj - this.delegate.taintInputObject(obj, source, mark) - logTaint obj - } - + @Override TaintedObject taint(Object obj, Range[] ranges) { objects << obj - this.delegate.taintInputString(obj, ranges) + final tainted = this.delegate.taint(obj, ranges) logTaint obj + return tainted } private final static Logger LOGGER = LoggerFactory.getLogger("map tainted objects") diff --git a/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/NoopOverheadController.groovy b/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/NoopOverheadController.groovy index 74964a13634..32152663dc1 100644 --- a/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/NoopOverheadController.groovy +++ b/dd-java-agent/agent-iast/src/testFixtures/groovy/com/datadog/iast/test/NoopOverheadController.groovy @@ -2,6 +2,7 @@ package com.datadog.iast.test import com.datadog.iast.overhead.Operation import com.datadog.iast.overhead.OverheadController +import com.github.javaparser.quality.Nullable import datadog.trace.bootstrap.instrumentation.api.AgentSpan import groovy.transform.CompileStatic @@ -18,12 +19,12 @@ class NoopOverheadController implements OverheadController { } @Override - boolean hasQuota(Operation operation, AgentSpan span) { + boolean hasQuota(Operation operation, @Nullable AgentSpan span) { true } @Override - boolean consumeQuota(Operation operation, AgentSpan span) { + boolean consumeQuota(Operation operation, @Nullable AgentSpan span) { true } diff --git a/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java b/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java index afa838ae20a..1958c015f65 100644 --- a/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java +++ b/dd-java-agent/agent-logging/src/main/java/datadog/trace/logging/ddlogger/DDLoggerFactory.java @@ -75,4 +75,9 @@ private LoggerHelperFactory getHelperFactory() { } return factory; } + + @Override + public void reinitialize() { + helperFactory = null; + } } diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle b/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle index 5de37b3baaa..2f561175283 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/build.gradle @@ -17,6 +17,9 @@ apply plugin: 'idea' dependencies { api deps.slf4j api project(':internal-api') + api(project(':dd-java-agent:agent-bootstrap')) { + exclude group: 'com.datadoghq', module: 'agent-logging' + } api project(':dd-java-agent:agent-profiling:profiling-auxiliary') api project(':dd-java-agent:agent-profiling:profiling-controller') api project(':dd-java-agent:agent-profiling:profiling-controller-jfr') diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java index 1f576bac76e..541ad63d14e 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java @@ -16,7 +16,10 @@ package com.datadog.profiling.controller.openjdk; import static com.datadog.profiling.controller.ProfilingSupport.*; +import static com.datadog.profiling.controller.ProfilingSupport.isObjectCountParallelized; import static datadog.trace.api.Platform.isJavaVersionAtLeast; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_HEAP_HISTOGRAM_ENABLED; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_HEAP_HISTOGRAM_ENABLED_DEFAULT; import static datadog.trace.api.config.ProfilingConfig.PROFILING_ULTRA_MINIMAL; import com.datadog.profiling.controller.ConfigurationException; @@ -27,6 +30,7 @@ import datadog.trace.api.Platform; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.bootstrap.instrumentation.jfr.exceptions.ExceptionProfiling; import de.thetaphi.forbiddenapis.SuppressForbidden; import java.io.IOException; import java.time.Duration; @@ -99,6 +103,16 @@ public OpenJdkController(final ConfigProvider configProvider) disableEvent(recordingSettings, "jdk.FileWrite", EXPENSIVE_ON_CURRENT_JVM); } + if (configProvider.getBoolean( + PROFILING_HEAP_HISTOGRAM_ENABLED, PROFILING_HEAP_HISTOGRAM_ENABLED_DEFAULT)) { + if (!isObjectCountParallelized()) { + log.warn( + "enabling Datadog heap histogram on JVM without an efficient implementation of the jdk.ObjectCount event. " + + "This may increase p99 latency. Consider upgrading to JDK 17.0.9+ or 21+ to reduce latency impact."); + } + enableEvent(recordingSettings, "jdk.ObjectCount", "user enabled histogram heap collection"); + } + // Toggle settings from override file try { @@ -169,6 +183,10 @@ && isEventEnabled(recordingSettings, "jdk.NativeMethodSample")) { this.recordingSettings = Collections.unmodifiableMap(recordingSettings); + if (isEventEnabled(this.recordingSettings, "datadog.ExceptionSample")) { + ExceptionProfiling.getInstance().start(); + } + // Register periodic events AvailableProcessorCoresEvent.register(); } diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilingSupport.java b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilingSupport.java index c53e8130947..5e04651f115 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilingSupport.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilingSupport.java @@ -26,6 +26,12 @@ public static boolean isObjectAllocationSampleAvailable() { return isJavaVersionAtLeast(16); } + public static boolean isObjectCountParallelized() { + // parallelized jdk.ObjectCount implemented in JDK21 and backported to JDK17 + // https://bugs.openjdk.org/browse/JDK-8307348 + return (isJavaVersion(17) && isJavaVersionAtLeast(17, 0, 9)) || isJavaVersionAtLeast(21); + } + public static boolean isNativeMethodSampleAvailable() { if (isOracleJDK8()) { return false; diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index 0178fb76d3b..bd8d4a37b24 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -16,6 +16,7 @@ import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isCpuProfilerEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isLiveHeapSizeTrackingEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isMemoryLeakProfilingEnabled; +import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isResourceNameContextAttributeEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isSpanNameContextAttributeEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.isWallClockProfilerEnabled; import static com.datadog.profiling.ddprof.DatadogProfilerConfig.omitLineNumbers; @@ -58,6 +59,7 @@ public final class DatadogProfiler { private static final int[] EMPTY = new int[0]; private static final String OPERATION = "_dd.trace.operation"; + private static final String RESOURCE = "_dd.trace.resource"; private static final int MAX_NUM_ENDPOINTS = 8192; @@ -173,6 +175,9 @@ private DatadogProfiler(ConfigProvider configProvider) throws UnsupportedEnviron if (isSpanNameContextAttributeEnabled(configProvider)) { orderedContextAttributes.add(OPERATION); } + if (isResourceNameContextAttributeEnabled(configProvider)) { + orderedContextAttributes.add(RESOURCE); + } this.contextSetter = new ContextSetter(profiler, orderedContextAttributes); this.queueTimeThreshold = configProvider.getLong( @@ -367,6 +372,10 @@ public int operationNameOffset() { return offsetOf(OPERATION); } + public int resourceNameOffset() { + return offsetOf(RESOURCE); + } + public int offsetOf(String attribute) { return contextSetter.offsetOf(attribute); } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java index d6eeac40899..3746ca29936 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilerConfig.java @@ -3,6 +3,7 @@ import static datadog.trace.api.Platform.isJ9; import static datadog.trace.api.config.ProfilingConfig.PROFILING_ALLOCATION_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES; +import static datadog.trace.api.config.ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES_RESOURCE_NAME_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_CONTEXT_ATTRIBUTES_SPAN_NAME_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_ALLOC_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DATADOG_PROFILER_ALLOC_ENABLED_DEFAULT; @@ -289,6 +290,10 @@ public static boolean isSpanNameContextAttributeEnabled(ConfigProvider configPro return configProvider.getBoolean(PROFILING_CONTEXT_ATTRIBUTES_SPAN_NAME_ENABLED, true); } + public static boolean isResourceNameContextAttributeEnabled(ConfigProvider configProvider) { + return configProvider.getBoolean(PROFILING_CONTEXT_ATTRIBUTES_RESOURCE_NAME_ENABLED, false); + } + public static String getString(ConfigProvider configProvider, String key, String defaultValue) { return configProvider.getString(key, configProvider.getString(normalizeKey(key), defaultValue)); } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java index 1fe10273789..54fe460b797 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfilingIntegration.java @@ -13,12 +13,10 @@ public class DatadogProfilingIntegration implements ProfilingContextIntegration private static final DatadogProfiler DDPROF = DatadogProfiler.getInstance(); private static final int SPAN_NAME_INDEX = DDPROF.operationNameOffset(); + private static final int RESOURCE_NAME_INDEX = DDPROF.resourceNameOffset(); private static final boolean WALLCLOCK_ENABLED = DatadogProfilerConfig.isWallClockProfilerEnabled(); - private static final boolean QUEUEING_TIME_ENABLED = - WALLCLOCK_ENABLED && DatadogProfilerConfig.isQueueingTimeEnabled(); - @Override public void onAttach() { if (WALLCLOCK_ENABLED) { @@ -42,12 +40,14 @@ public int encode(CharSequence constant) { public void setContext(ProfilerContext profilerContext) { DDPROF.setSpanContext(profilerContext.getSpanId(), profilerContext.getRootSpanId()); DDPROF.setContextValue(SPAN_NAME_INDEX, profilerContext.getEncodedOperationName()); + DDPROF.setContextValue(RESOURCE_NAME_INDEX, profilerContext.getEncodedResourceName()); } @Override public void clearContext() { DDPROF.clearSpanContext(); DDPROF.clearContextValue(SPAN_NAME_INDEX); + DDPROF.clearContextValue(RESOURCE_NAME_INDEX); } @Override @@ -64,12 +64,4 @@ public ProfilingContextAttribute createContextAttribute(String attribute) { public ProfilingScope newScope() { return new DatadogProfilingScope(DDPROF); } - - @Override - public boolean isQueuingTimeEnabled() { - return QUEUEING_TIME_ENABLED; - } - - @Override - public void recordQueueingTime(long duration) {} } diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/matcher/ScalaTraitMatchers.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/matcher/ScalaTraitMatchers.java new file mode 100644 index 00000000000..51464848f33 --- /dev/null +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/matcher/ScalaTraitMatchers.java @@ -0,0 +1,44 @@ +package datadog.trace.agent.tooling.bytebuddy.matcher; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.is; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isStatic; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +public class ScalaTraitMatchers { + public static ElementMatcher.Junction isTraitMethod( + String traitName, String name, Object... argumentTypes) { + + ElementMatcher.Junction scalaOldArgs = + isStatic() + .and(takesArguments(argumentTypes.length + 1)) + .and(takesArgument(0, named(traitName))); + ElementMatcher.Junction scalaNewArgs = + not(isStatic()).and(takesArguments(argumentTypes.length)); + + for (int i = 0; i < argumentTypes.length; i++) { + Object argumentType = argumentTypes[i]; + ElementMatcher matcher; + if (argumentType instanceof ElementMatcher) { + matcher = (ElementMatcher) argumentType; + } else if (argumentType instanceof String) { + matcher = named((String) argumentType); + } else if (argumentType instanceof Class) { + matcher = is((Class) argumentType); + } else { + throw new IllegalArgumentException("Unexpected type for argument type specification"); + } + scalaOldArgs = scalaOldArgs.and(takesArgument(i + 1, matcher)); + scalaNewArgs = scalaNewArgs.and(takesArgument(i, matcher)); + } + + return isMethod().and(named(name)).and(scalaOldArgs.or(scalaNewArgs)); + } +} diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/iast/TaintableEnumeration.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/iast/TaintableEnumeration.java new file mode 100644 index 00000000000..cfc8fb3b44f --- /dev/null +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/iast/TaintableEnumeration.java @@ -0,0 +1,101 @@ +package datadog.trace.agent.tooling.iast; + +import datadog.trace.api.iast.IastContext; +import datadog.trace.api.iast.propagation.PropagationModule; +import datadog.trace.util.stacktrace.StackUtils; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Enumeration; +import javax.annotation.Nullable; + +public class TaintableEnumeration implements Enumeration { + + private static final String CLASS_NAME = TaintableEnumeration.class.getName(); + + private volatile IastContext context; + private volatile boolean contextFetched; + + private final PropagationModule module; + + private final byte origin; + + private final CharSequence name; + + private final boolean useValueAsName; + + private final Enumeration delegate; + + private TaintableEnumeration( + @NonNull final Enumeration delegate, + @NonNull final PropagationModule module, + final byte origin, + @Nullable final CharSequence name, + final boolean useValueAsName) { + this.delegate = delegate; + this.module = module; + this.origin = origin; + this.name = name; + this.useValueAsName = useValueAsName; + } + + @Override + public boolean hasMoreElements() { + try { + return delegate.hasMoreElements(); + } catch (Throwable e) { + StackUtils.filterFirst(e, TaintableEnumeration::nonTaintableEnumerationStack); + throw e; + } + } + + @Override + public String nextElement() { + final String next; + try { + next = delegate.nextElement(); + } catch (Throwable e) { + StackUtils.filterFirst(e, TaintableEnumeration::nonTaintableEnumerationStack); + throw e; + } + try { + module.taint(context(), next, origin, name(next)); + } catch (final Throwable e) { + module.onUnexpectedException("Failed to taint enumeration", e); + } + return next; + } + + private IastContext context() { + if (!contextFetched) { + contextFetched = true; + context = IastContext.Provider.get(); + } + return context; + } + + private CharSequence name(final String value) { + if (name != null) { + return name; + } + return useValueAsName ? value : null; + } + + private static boolean nonTaintableEnumerationStack(final StackTraceElement element) { + return !CLASS_NAME.equals(element.getClassName()); + } + + public static Enumeration wrap( + @NonNull final Enumeration delegate, + @NonNull final PropagationModule module, + final byte origin, + @Nullable final CharSequence name) { + return new TaintableEnumeration(delegate, module, origin, name, false); + } + + public static Enumeration wrap( + @NonNull final Enumeration delegate, + @NonNull final PropagationModule module, + final byte origin, + boolean useValueAsName) { + return new TaintableEnumeration(delegate, module, origin, null, useValueAsName); + } +} diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index fd259e3cd80..d244f5a623b 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -42,7 +42,8 @@ 1 io.opentelemetry.javaagent.* 1 java.* # allow exception profiling instrumentation -0 java.lang.Throwable +0 java.lang.Exception +0 java.lang.Error # allow ProcessImpl instrumentation 0 java.lang.ProcessImpl 0 java.net.http.* @@ -192,6 +193,9 @@ 0 com.google.common.base.internal.Finalizer 0 com.google.common.util.concurrent.* 2 com.google.gson.* +# Need for IAST: we instrument this class +0 com.google.gson.Gson +0 com.google.gson.stream.JsonReader 2 com.google.inject.* # We instrument Runnable there 0 com.google.inject.internal.AbstractBindingProcessor$* @@ -284,8 +288,6 @@ 2 org.springframework.expression.* 2 org.springframework.format.* 2 org.springframework.http.* -# Need for IAST: calls ServletRequest methods instrumented at callsite -0 org.springframework.http.server.ServletServerHttpRequest # Need for IAST: we instrument these classes 0 org.springframework.http.HttpHeaders 0 org.springframework.http.ReadOnlyHttpHeaders @@ -321,14 +323,11 @@ 2 org.springframework.validation.* 2 org.springframework.web.* 0 org.springframework.web.context.request.async.* -0 org.springframework.web.context.request.* 0 org.springframework.web.context.support.AbstractRefreshableWebApplicationContext 0 org.springframework.web.context.support.GenericWebApplicationContext 0 org.springframework.web.context.support.XmlWebApplicationContext 0 org.springframework.web.reactive.* 0 org.springframework.web.servlet.* -# Included for IAST -0 org.springframework.web.util.WebUtils # Need for IAST so propagation of tainted objects is complete in spring 2.7.5 0 org.springframework.util.StreamUtils$NonClosingInputStream # Included for IAST Spring mvc unvalidated redirect and xss vulnerability diff --git a/dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/iast/TaintableEnumerationTest.groovy b/dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/iast/TaintableEnumerationTest.groovy new file mode 100644 index 00000000000..78566a8bec7 --- /dev/null +++ b/dd-java-agent/agent-tooling/src/test/groovy/datadog/trace/agent/tooling/iast/TaintableEnumerationTest.groovy @@ -0,0 +1,99 @@ +package datadog.trace.agent.tooling.iast + +import datadog.trace.api.gateway.RequestContext +import datadog.trace.api.gateway.RequestContextSlot +import datadog.trace.api.iast.IastContext +import datadog.trace.api.iast.InstrumentationBridge +import datadog.trace.api.iast.SourceTypes +import datadog.trace.api.iast.propagation.PropagationModule +import datadog.trace.bootstrap.instrumentation.api.AgentSpan +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.test.util.DDSpecification +import spock.lang.Shared + +class TaintableEnumerationTest extends DDSpecification { + + @Shared + protected static final AgentTracer.TracerAPI ORIGINAL_TRACER = AgentTracer.get() + + protected AgentTracer.TracerAPI tracer = Mock(AgentTracer.TracerAPI) + + protected IastContext iastCtx = Mock(IastContext) + + protected RequestContext reqCtx = Mock(RequestContext) { + getData(RequestContextSlot.IAST) >> iastCtx + } + + protected AgentSpan span = Mock(AgentSpan) { + getRequestContext() >> reqCtx + } + + protected PropagationModule module + + + void setup() { + AgentTracer.forceRegister(tracer) + module = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(module) + } + + void cleanup() { + AgentTracer.forceRegister(ORIGINAL_TRACER) + InstrumentationBridge.clearIastModules() + } + + void 'underlying enumerated values are tainted with a name'() { + given: + final values = (1..10).collect { "value$it".toString() } + final origin = SourceTypes.REQUEST_PARAMETER_NAME + final name = 'test' + final enumeration = TaintableEnumeration.wrap(Collections.enumeration(values), module, origin, name) + + when: + final result = enumeration.collect() + + then: + result == values + values.each { 1 * module.taint(_, it, origin, name) } + 1 * tracer.activeSpan() >> span // only one access to the active context + } + + void 'underlying enumerated values are tainted with the value as a name'() { + given: + final values = (1..10).collect { "value$it".toString() } + final origin = SourceTypes.REQUEST_PARAMETER_NAME + final enumeration = TaintableEnumeration.wrap(Collections.enumeration(values), module, origin, true) + + when: + final result = enumeration.collect() + + then: + result == values + values.each { 1 * module.taint(_, it, origin, it) } + } + + void 'taintable enumeration leaves no trace in case of error'() { + given: + final origin = SourceTypes.REQUEST_PARAMETER_NAME + final enumeration = TaintableEnumeration.wrap(new BadEnumeration(), module, origin, true) + + when: + enumeration.hasMoreElements() + + then: + final first = thrown(Error) + first.stackTrace.find { it.className == TaintableEnumeration.name } == null + } + + private static class BadEnumeration implements Enumeration { + @Override + boolean hasMoreElements() { + throw new Error('Ooops!!!') + } + + @Override + String nextElement() { + throw new Error('Boom!!!') + } + } +} diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpServerHeaders.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpServerHeaders.java index 589bc3d81b4..c5aabac9926 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpServerHeaders.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpServerHeaders.java @@ -1,6 +1,7 @@ package datadog.trace.instrumentation.akkahttp; import akka.http.javadsl.model.HttpHeader; +import akka.http.javadsl.model.headers.RawRequestURI; import akka.http.javadsl.model.headers.RemoteAddress; import akka.http.javadsl.model.headers.TimeoutAccess; import akka.http.scaladsl.model.ContentType; @@ -45,7 +46,9 @@ private static void doForEachKey( for (final HttpHeader header : carrier.getHeaders()) { // skip synthetic headers - if (header instanceof RemoteAddress || header instanceof TimeoutAccess) { + if (header instanceof RemoteAddress + || header instanceof TimeoutAccess + || header instanceof RawRequestURI) { continue; } if (!classifier.accept(header.lowercaseName(), header.value())) { diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/BlockingResponseHelper.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/BlockingResponseHelper.java index 064448c764f..046d6c96bdb 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/BlockingResponseHelper.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/BlockingResponseHelper.java @@ -10,6 +10,7 @@ import akka.http.scaladsl.model.HttpResponse; import akka.http.scaladsl.model.ResponseEntity; import akka.http.scaladsl.model.StatusCode; +import akka.http.scaladsl.model.StatusCodes; import akka.util.ByteString; import datadog.appsec.api.blocking.BlockingContentType; import datadog.trace.api.gateway.BlockResponseFunction; @@ -95,7 +96,12 @@ public static HttpResponse maybeCreateBlockingResponse( RawHeader.create(e.getKey(), e.getValue())) .collect(ScalaListCollector.toScalaList()); - return HttpResponse.apply( - StatusCode.int2StatusCode(httpCode), headersList, entity, request.protocol()); + StatusCode code; + try { + code = StatusCode.int2StatusCode(httpCode); + } catch (RuntimeException e) { + code = StatusCodes.custom(httpCode, "Request Blocked", "", false, true); + } + return HttpResponse.apply(code, headersList, entity, request.protocol()); } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/MultipartUnmarshallersInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/MultipartUnmarshallersInstrumentation.java index b631cb87154..ab6e90c1517 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/MultipartUnmarshallersInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/MultipartUnmarshallersInstrumentation.java @@ -1,7 +1,7 @@ package datadog.trace.instrumentation.akkahttp.appsec; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static datadog.trace.instrumentation.akkahttp.iast.TraitMethodMatchers.isTraitMethod; +import static datadog.trace.agent.tooling.bytebuddy.matcher.ScalaTraitMatchers.isTraitMethod; import static net.bytebuddy.matcher.ElementMatchers.returns; import akka.http.scaladsl.unmarshalling.MultipartUnmarshallers; diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/PredefinedFromEntityUnmarshallersInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/PredefinedFromEntityUnmarshallersInstrumentation.java index 0cda26c28bf..ab01b6c94a3 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/PredefinedFromEntityUnmarshallersInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/PredefinedFromEntityUnmarshallersInstrumentation.java @@ -2,7 +2,7 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.namedOneOf; -import static datadog.trace.instrumentation.akkahttp.iast.TraitMethodMatchers.isTraitMethod; +import static datadog.trace.agent.tooling.bytebuddy.matcher.ScalaTraitMatchers.isTraitMethod; import static net.bytebuddy.matcher.ElementMatchers.returns; import akka.http.scaladsl.unmarshalling.PredefinedFromEntityUnmarshallers; diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/SprayUnmarshallerInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/SprayUnmarshallerInstrumentation.java index eeb85b694ee..2e5aa79515f 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/SprayUnmarshallerInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/appsec/SprayUnmarshallerInstrumentation.java @@ -1,7 +1,7 @@ package datadog.trace.instrumentation.akkahttp.appsec; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static datadog.trace.instrumentation.akkahttp.iast.TraitMethodMatchers.isTraitMethod; +import static datadog.trace.agent.tooling.bytebuddy.matcher.ScalaTraitMatchers.isTraitMethod; import static net.bytebuddy.matcher.ElementMatchers.returns; import akka.http.scaladsl.unmarshalling.Unmarshaller; diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/CookieHeaderInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/CookieHeaderInstrumentation.java index fa9e098b81e..1c8b35e4bb3 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/CookieHeaderInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/CookieHeaderInstrumentation.java @@ -12,13 +12,11 @@ import akka.http.scaladsl.model.headers.HttpCookiePair; import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.Source; import datadog.trace.api.iast.SourceTypes; import datadog.trace.api.iast.propagation.PropagationModule; -import datadog.trace.api.iast.source.WebModule; -import java.util.ArrayList; -import java.util.List; import net.bytebuddy.asm.Advice; import scala.collection.Iterator; import scala.collection.immutable.Seq; @@ -56,23 +54,22 @@ static class TaintAllCookiesAdvice { @Source(SourceTypes.REQUEST_COOKIE_VALUE) static void after( @Advice.This HttpHeader cookie, @Advice.Return Seq cookiePairs) { - WebModule mod = InstrumentationBridge.WEB; PropagationModule prop = InstrumentationBridge.PROPAGATION; - if (mod == null || prop == null || cookiePairs == null || cookiePairs.isEmpty()) { + if (prop == null || cookiePairs == null || cookiePairs.isEmpty()) { return; } if (!prop.isTainted(cookie)) { return; } + final IastContext ctx = IastContext.Provider.get(); Iterator iterator = cookiePairs.iterator(); - List cookieNames = new ArrayList<>(); while (iterator.hasNext()) { HttpCookiePair pair = iterator.next(); - cookieNames.add(pair.name()); - prop.taint(SourceTypes.REQUEST_COOKIE_VALUE, pair.name(), pair.value()); + final String name = pair.name(), value = pair.value(); + prop.taint(ctx, name, SourceTypes.REQUEST_COOKIE_NAME, name); + prop.taint(ctx, value, SourceTypes.REQUEST_COOKIE_VALUE, name); } - mod.onCookieNames(cookieNames); } } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HeaderNameCallSite.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HeaderNameCallSite.java index 6122b91b361..f7f330a14b5 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HeaderNameCallSite.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HeaderNameCallSite.java @@ -26,7 +26,7 @@ public static String after(@CallSite.This HttpHeader header, @CallSite.Return St return result; } try { - module.taintIfInputIsTainted(SourceTypes.REQUEST_HEADER_NAME, result, result, header); + module.taintIfTainted(result, header, SourceTypes.REQUEST_HEADER_NAME, result); } catch (final Throwable e) { module.onUnexpectedException("onHeaderNames threw", e); } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpHeaderSubclassesInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpHeaderSubclassesInstrumentation.java index 2c9b5cb507b..57cd04ce127 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpHeaderSubclassesInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpHeaderSubclassesInstrumentation.java @@ -62,7 +62,7 @@ static void onExit(@Advice.This HttpHeader h, @Advice.Return String retVal) { return; } - propagation.taintIfInputIsTainted(retVal, h); + propagation.taintIfTainted(retVal, h); } } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpRequestInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpRequestInstrumentation.java index 16b124624f8..a0c5f432e85 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpRequestInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/HttpRequestInstrumentation.java @@ -13,6 +13,7 @@ import akka.http.scaladsl.model.HttpRequest; import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.Propagation; import datadog.trace.api.iast.Source; @@ -70,6 +71,7 @@ static void onExit( return; } + final IastContext ctx = IastContext.Provider.get(); Iterator iterator = headers.iterator(); while (iterator.hasNext()) { HttpHeader h = iterator.next(); @@ -78,7 +80,7 @@ static void onExit( } // unfortunately, the call to h.value() is instrumented, but // because the call to taint() only happens after, the call is a noop - propagation.taintObject(SourceTypes.REQUEST_HEADER_VALUE, h.name(), h.value(), h); + propagation.taint(ctx, h, SourceTypes.REQUEST_HEADER_VALUE, h.name(), h.value()); } } } @@ -98,7 +100,7 @@ static void onExit( return; } - propagation.taintIfInputIsTainted(entity, thiz); + propagation.taintIfTainted(entity, thiz); } } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/PathMatcherInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/PathMatcherInstrumentation.java index 1a4dcf27ce2..80c3b2687f0 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/PathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/PathMatcherInstrumentation.java @@ -11,6 +11,7 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.api.gateway.RequestContext; import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.Source; import datadog.trace.api.iast.SourceTypes; @@ -68,11 +69,8 @@ static void onExit( } if (value instanceof String) { - module.taint( - reqCtx.getData(RequestContextSlot.IAST), - SourceTypes.REQUEST_PATH_PARAMETER, - null, - (String) value); + final IastContext ctx = reqCtx.getData(RequestContextSlot.IAST); + module.taint(ctx, value, SourceTypes.REQUEST_PATH_PARAMETER); } } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/RequestContextInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/RequestContextInstrumentation.java index 0cdd0266796..f18148a7ceb 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/RequestContextInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/RequestContextInstrumentation.java @@ -53,7 +53,7 @@ static void onExit( return; } - propagation.taintIfInputIsTainted(request, requestContext); + propagation.taintIfTainted(request, requestContext); } } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/TraitMethodMatchers.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/TraitMethodMatchers.java index d9db76ece90..fb58538102e 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/TraitMethodMatchers.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/TraitMethodMatchers.java @@ -1,46 +1,15 @@ package datadog.trace.instrumentation.akkahttp.iast; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isStatic; -import static net.bytebuddy.matcher.ElementMatchers.not; +import static datadog.trace.agent.tooling.bytebuddy.matcher.ScalaTraitMatchers.isTraitMethod; import static net.bytebuddy.matcher.ElementMatchers.returns; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import net.bytebuddy.description.method.MethodDescription; -import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; public class TraitMethodMatchers { - public static ElementMatcher.Junction isTraitMethod( - String traitName, String name, Object... argumentTypes) { - - ElementMatcher.Junction scalaOldArgs = - isStatic() - .and(takesArguments(argumentTypes.length + 1)) - .and(takesArgument(0, named(traitName))); - ElementMatcher.Junction scalaNewArgs = - not(isStatic()).and(takesArguments(argumentTypes.length)); - - for (int i = 0; i < argumentTypes.length; i++) { - Object argumentType = argumentTypes[i]; - ElementMatcher matcher; - if (argumentType instanceof ElementMatcher) { - matcher = (ElementMatcher) argumentType; - } else { - matcher = named((String) argumentType); - } - scalaOldArgs = scalaOldArgs.and(takesArgument(i + 1, matcher)); - scalaNewArgs = scalaNewArgs.and(takesArgument(i, matcher)); - } - - return isMethod().and(named(name)).and(scalaOldArgs.or(scalaNewArgs)); - } - public static ElementMatcher.Junction isTraitDirectiveMethod( - String traitName, String name, String... argumentTypes) { - + String traitName, String name, Object... argumentTypes) { return isTraitMethod(traitName, name, (Object[]) argumentTypes) .and(returns(named("akka.http.scaladsl.server.Directive"))); } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/UriInstrumentation.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/UriInstrumentation.java index 9faab16a32b..49a03c9bf32 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/UriInstrumentation.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/UriInstrumentation.java @@ -11,13 +11,12 @@ import akka.http.scaladsl.model.Uri; import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.Propagation; import datadog.trace.api.iast.Source; import datadog.trace.api.iast.SourceTypes; import datadog.trace.api.iast.propagation.PropagationModule; -import datadog.trace.api.iast.source.WebModule; -import java.util.Collections; import net.bytebuddy.asm.Advice; import scala.Tuple2; import scala.collection.Iterator; @@ -70,7 +69,7 @@ static void after(@Advice.This Uri uri, @Advice.Return scala.Option ret) if (mod == null || ret.isEmpty()) { return; } - mod.taintIfInputIsTainted(ret.get(), uri); + mod.taintIfTainted(ret.get(), uri); } } @@ -80,9 +79,8 @@ public static class TaintQueryAdvice { @Advice.OnMethodExit(suppress = Throwable.class) @Source(SourceTypes.REQUEST_PARAMETER_VALUE) static void after(@Advice.This /*Uri*/ Object uri, @Advice.Return Uri.Query ret) { - WebModule web = InstrumentationBridge.WEB; PropagationModule prop = InstrumentationBridge.PROPAGATION; - if (prop == null || web == null || ret.isEmpty()) { + if (prop == null || ret.isEmpty()) { return; } @@ -90,11 +88,13 @@ static void after(@Advice.This /*Uri*/ Object uri, @Advice.Return Uri.Query ret) return; } + final IastContext ctx = IastContext.Provider.get(); Iterator> iterator = ret.iterator(); while (iterator.hasNext()) { Tuple2 pair = iterator.next(); - web.onParameterNames(Collections.singleton(pair._1())); - prop.taint(SourceTypes.REQUEST_PARAMETER_VALUE, pair._1(), pair._2()); + final String name = pair._1(), value = pair._2(); + prop.taint(ctx, name, SourceTypes.REQUEST_PARAMETER_NAME, name); + prop.taint(ctx, value, SourceTypes.REQUEST_PARAMETER_VALUE, name); } } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintCookieFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintCookieFunction.java index 2d9c23c51ba..0955a0a9e49 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintCookieFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintCookieFunction.java @@ -16,10 +16,13 @@ public Tuple1 apply(Tuple1 v1) { HttpCookiePair httpCookiePair = v1._1(); PropagationModule mod = InstrumentationBridge.PROPAGATION; - if (mod == null) { + if (mod == null || httpCookiePair == null) { return v1; } - mod.taint(SourceTypes.REQUEST_COOKIE_VALUE, httpCookiePair.name(), httpCookiePair.value()); + final String name = httpCookiePair.name(); + final String value = httpCookiePair.value(); + mod.taint(name, SourceTypes.REQUEST_COOKIE_NAME, name); + mod.taint(value, SourceTypes.REQUEST_COOKIE_VALUE, name); return v1; } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintFutureHelper.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintFutureHelper.java index 922397f52f6..c1ff9dbc8c2 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintFutureHelper.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintFutureHelper.java @@ -10,7 +10,7 @@ public static Future wrapFuture( Future f, Object input, PropagationModule mod, ExecutionContext ec) { JFunction1 mapf = t -> { - mod.taintIfInputIsTainted(t, input); + mod.taintIfTainted(t, input); return t; }; return f.map(mapf, ec); diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMapFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMapFunction.java index 40d75a9e15f..705b1921742 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMapFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMapFunction.java @@ -1,9 +1,9 @@ package datadog.trace.instrumentation.akkahttp.iast.helpers; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.SourceTypes; import datadog.trace.api.iast.propagation.PropagationModule; -import datadog.trace.api.iast.source.WebModule; import scala.Tuple1; import scala.Tuple2; import scala.collection.Iterator; @@ -19,18 +19,17 @@ public Tuple1> apply(Tuple1> v1) { Map m = v1._1; PropagationModule prop = InstrumentationBridge.PROPAGATION; - WebModule web = InstrumentationBridge.WEB; - if (web == null || prop == null || m == null) { + if (prop == null || m == null || m.isEmpty()) { return v1; } - java.util.List keysAsCollection = ScalaToJava.keySetAsCollection(m); - web.onParameterNames(keysAsCollection); - + final IastContext ctx = IastContext.Provider.get(); Iterator> iterator = m.iterator(); while (iterator.hasNext()) { Tuple2 e = iterator.next(); - prop.taint(SourceTypes.REQUEST_PARAMETER_VALUE, e._1(), e._2()); + final String name = e._1(), value = e._2(); + prop.taint(ctx, name, SourceTypes.REQUEST_PARAMETER_NAME, name); + prop.taint(ctx, value, SourceTypes.REQUEST_PARAMETER_VALUE, name); } return v1; diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMultiMapFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMultiMapFunction.java index dcfe919d598..b72ac179c0b 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMultiMapFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintMultiMapFunction.java @@ -1,7 +1,9 @@ package datadog.trace.instrumentation.akkahttp.iast.helpers; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; -import datadog.trace.api.iast.source.WebModule; +import datadog.trace.api.iast.SourceTypes; +import datadog.trace.api.iast.propagation.PropagationModule; import scala.Tuple1; import scala.Tuple2; import scala.collection.Iterator; @@ -17,19 +19,21 @@ public class TaintMultiMapFunction public Tuple1>> apply(Tuple1>> v1) { Map> m = v1._1; - WebModule mod = InstrumentationBridge.WEB; - if (mod == null || m == null) { + PropagationModule mod = InstrumentationBridge.PROPAGATION; + if (mod == null || m == null || m.isEmpty()) { return v1; } - java.util.List keysAsCollection = ScalaToJava.keySetAsCollection(m); - mod.onParameterNames(keysAsCollection); - + final IastContext ctx = IastContext.Provider.get(); Iterator>> entriesIterator = m.iterator(); while (entriesIterator.hasNext()) { Tuple2> e = entriesIterator.next(); + final String name = e._1(); + mod.taint(ctx, name, SourceTypes.REQUEST_PARAMETER_NAME, name); List values = e._2(); - mod.onParameterValues(e._1(), ScalaToJava.listAsList(values)); + for (final String value : ScalaToJava.listAsList(values)) { + mod.taint(ctx, value, SourceTypes.REQUEST_PARAMETER_VALUE, name); + } } return v1; diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintOptionalCookieFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintOptionalCookieFunction.java index 1f6d8bb62a2..620bea2ebf8 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintOptionalCookieFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintOptionalCookieFunction.java @@ -17,13 +17,14 @@ public Tuple1> apply(Tuple1> v1) { Option httpCookiePair = v1._1(); PropagationModule mod = InstrumentationBridge.PROPAGATION; - if (mod == null || httpCookiePair.isEmpty()) { + if (mod == null || httpCookiePair == null || httpCookiePair.isEmpty()) { return v1; } - mod.taint( - SourceTypes.REQUEST_COOKIE_VALUE, - httpCookiePair.get().name(), - httpCookiePair.get().value()); + final HttpCookiePair cookie = httpCookiePair.get(); + final String name = cookie.name(); + final String value = cookie.value(); + mod.taint(name, SourceTypes.REQUEST_COOKIE_NAME, name); + mod.taint(value, SourceTypes.REQUEST_COOKIE_VALUE, name); return v1; } } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestContextFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestContextFunction.java index c4c8cbb2d5c..bc86e937643 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestContextFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestContextFunction.java @@ -16,10 +16,10 @@ public Tuple1 apply(Tuple1 v1) { RequestContext reqCtx = v1._1(); PropagationModule mod = InstrumentationBridge.PROPAGATION; - if (mod == null) { + if (mod == null || reqCtx == null) { return v1; } - mod.taintObject(SourceTypes.REQUEST_BODY, reqCtx); + mod.taint(reqCtx, SourceTypes.REQUEST_BODY); return v1; } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestFunction.java index e1b9a5e4af4..7894326ab89 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintRequestFunction.java @@ -17,10 +17,10 @@ public Tuple1 apply(Tuple1 v1) { HttpRequest httpRequest = v1._1(); PropagationModule mod = InstrumentationBridge.PROPAGATION; - if (mod == null) { + if (mod == null || httpRequest == null) { return v1; } - mod.taintObject(SourceTypes.REQUEST_BODY, httpRequest); + mod.taint(httpRequest, SourceTypes.REQUEST_BODY); return v1; } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSeqFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSeqFunction.java index 0d41fc32e56..fdec8e35b4e 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSeqFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSeqFunction.java @@ -1,9 +1,9 @@ package datadog.trace.instrumentation.akkahttp.iast.helpers; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.SourceTypes; import datadog.trace.api.iast.propagation.PropagationModule; -import datadog.trace.api.iast.source.WebModule; import java.util.Collections; import java.util.IdentityHashMap; import java.util.Set; @@ -22,22 +22,22 @@ public class TaintSeqFunction public Tuple1>> apply(Tuple1>> v1) { Seq> seq = v1._1; - WebModule web = InstrumentationBridge.WEB; PropagationModule prop = InstrumentationBridge.PROPAGATION; - if (web == null || prop == null || seq == null) { + if (prop == null || seq == null || seq.isEmpty()) { return v1; } + final IastContext ctx = IastContext.Provider.get(); Iterator> iterator = seq.iterator(); - Set seenKeys = Collections.newSetFromMap(new IdentityHashMap()); + Set seenKeys = Collections.newSetFromMap(new IdentityHashMap<>()); while (iterator.hasNext()) { Tuple2 t = iterator.next(); String name = t._1(); String value = t._2(); if (seenKeys.add(name)) { - web.onParameterNames(Collections.singleton(name)); + prop.taint(ctx, name, SourceTypes.REQUEST_PARAMETER_NAME, name); } - prop.taint(SourceTypes.REQUEST_PARAMETER_VALUE, name, value); + prop.taint(ctx, value, SourceTypes.REQUEST_PARAMETER_VALUE, name); } return v1; diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSingleParameterFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSingleParameterFunction.java index e3130be8062..f6c1073fff8 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSingleParameterFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintSingleParameterFunction.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.akkahttp.iast.helpers; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.SourceTypes; import datadog.trace.api.iast.propagation.PropagationModule; @@ -40,15 +41,16 @@ public Tuple1 apply(Tuple1 v1) { } if (value instanceof Iterable) { - Iterator iterator = ((Iterable) value).iterator(); + final IastContext ctx = IastContext.Provider.get(); + Iterator iterator = ((Iterable) value).iterator(); while (iterator.hasNext()) { Object o = iterator.next(); if (o instanceof String) { - mod.taint(SourceTypes.REQUEST_PARAMETER_VALUE, paramName, (String) o); + mod.taint(ctx, (String) o, SourceTypes.REQUEST_PARAMETER_VALUE, paramName); } } } else if (value instanceof String) { - mod.taint(SourceTypes.REQUEST_PARAMETER_VALUE, paramName, (String) value); + mod.taint((String) value, SourceTypes.REQUEST_PARAMETER_VALUE, paramName); } return v1; diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUnmarshaller.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUnmarshaller.java index 88d1580643a..d109cc92d9d 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUnmarshaller.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUnmarshaller.java @@ -28,7 +28,7 @@ public TaintUnmarshaller(PropagationModule propagationModule, Unmarshaller @Override public Future apply(A value, ExecutionContext ec, Materializer materializer) { - propagationModule.taintObject(SourceTypes.REQUEST_BODY, value); + propagationModule.taint(value, SourceTypes.REQUEST_BODY); return delegate.apply(value, ec, materializer); } diff --git a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUriFunction.java b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUriFunction.java index d11830b047e..5feb8315119 100644 --- a/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUriFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/iast/helpers/TaintUriFunction.java @@ -18,7 +18,7 @@ public Tuple1 apply(Tuple1 v1) { if (mod == null) { return v1; } - mod.taintObject(SourceTypes.REQUEST_QUERY, uri); + mod.taint(uri, SourceTypes.REQUEST_QUERY); return v1; } diff --git a/dd-java-agent/instrumentation/akka-http-10.2-iast/src/main/java/datadog/trace/instrumentation/akkahttp102/iast/helpers/TaintParametersFunction.java b/dd-java-agent/instrumentation/akka-http-10.2-iast/src/main/java/datadog/trace/instrumentation/akkahttp102/iast/helpers/TaintParametersFunction.java index 213ee7638e1..87008ac5ab4 100644 --- a/dd-java-agent/instrumentation/akka-http-10.2-iast/src/main/java/datadog/trace/instrumentation/akkahttp102/iast/helpers/TaintParametersFunction.java +++ b/dd-java-agent/instrumentation/akka-http-10.2-iast/src/main/java/datadog/trace/instrumentation/akkahttp102/iast/helpers/TaintParametersFunction.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.akkahttp102.iast.helpers; +import datadog.trace.api.iast.IastContext; import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.SourceTypes; import datadog.trace.api.iast.propagation.PropagationModule; @@ -33,15 +34,16 @@ public Tuple1 apply(Tuple1 v1) { } if (value instanceof Iterable) { - Iterator iterator = ((Iterable) value).iterator(); + final IastContext ctx = IastContext.Provider.get(); + Iterator iterator = ((Iterable) value).iterator(); while (iterator.hasNext()) { Object o = iterator.next(); if (o instanceof String) { - mod.taint(SourceTypes.REQUEST_PARAMETER_VALUE, paramName, (String) o); + mod.taint(ctx, (String) o, SourceTypes.REQUEST_PARAMETER_VALUE, paramName); } } } else if (value instanceof String) { - mod.taint(SourceTypes.REQUEST_PARAMETER_VALUE, paramName, (String) value); + mod.taint((String) value, SourceTypes.REQUEST_PARAMETER_VALUE, paramName); } return v1; diff --git a/dd-java-agent/instrumentation/armeria-grpc/src/main/java/datadog/trace/instrumentation/armeria/grpc/server/TracingServerInterceptor.java b/dd-java-agent/instrumentation/armeria-grpc/src/main/java/datadog/trace/instrumentation/armeria/grpc/server/TracingServerInterceptor.java index 9e7aed6b735..158c7c72d01 100644 --- a/dd-java-agent/instrumentation/armeria-grpc/src/main/java/datadog/trace/instrumentation/armeria/grpc/server/TracingServerInterceptor.java +++ b/dd-java-agent/instrumentation/armeria-grpc/src/main/java/datadog/trace/instrumentation/armeria/grpc/server/TracingServerInterceptor.java @@ -68,7 +68,9 @@ public ServerCall.Listener interceptCall( final AgentSpan span = startSpan(DECORATE.instrumentationNames()[0], GRPC_SERVER, spanContext).setMeasured(true); - AgentTracer.get().getDataStreamsMonitoring().setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0); + AgentTracer.get() + .getDataStreamsMonitoring() + .setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0, 0); RequestContext reqContext = span.getRequestContext(); if (reqContext != null) { diff --git a/dd-java-agent/instrumentation/commons-httpclient-2/src/main/java/datadog/trace/instrumentation/commonshttpclient/IastHttpMethodBaseInstrumentation.java b/dd-java-agent/instrumentation/commons-httpclient-2/src/main/java/datadog/trace/instrumentation/commonshttpclient/IastHttpMethodBaseInstrumentation.java index 8371ad69442..11522f224fd 100644 --- a/dd-java-agent/instrumentation/commons-httpclient-2/src/main/java/datadog/trace/instrumentation/commonshttpclient/IastHttpMethodBaseInstrumentation.java +++ b/dd-java-agent/instrumentation/commons-httpclient-2/src/main/java/datadog/trace/instrumentation/commonshttpclient/IastHttpMethodBaseInstrumentation.java @@ -51,7 +51,7 @@ public static void afterCtor( @Advice.This final Object self, @Advice.Argument(0) final Object argument) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { - module.taintIfInputIsTainted(self, argument); + module.taintIfTainted(self, argument); } } } diff --git a/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/IastCommonsHttpClientInstrumentationTest.groovy b/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/IastCommonsHttpClientInstrumentationTest.groovy index 3624973f60e..0a3f5ba5bc7 100644 --- a/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/IastCommonsHttpClientInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/commons-httpclient-2/src/test/groovy/IastCommonsHttpClientInstrumentationTest.groovy @@ -51,7 +51,7 @@ class IastCommonsHttpClientInstrumentationTest extends AgentTestRunner { private void mockPropagation() { final propagation = Mock(PropagationModule) { - taintIfInputIsTainted(_, _) >> { + taintIfTainted(_, _) >> { if (tainteds.containsKey(it[1])) { tainteds.put(it[0], null) } diff --git a/dd-java-agent/instrumentation/commons-lang-2/src/main/java/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSite.java b/dd-java-agent/instrumentation/commons-lang-2/src/main/java/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSite.java index 63e71ac0325..d7fdd84417d 100644 --- a/dd-java-agent/instrumentation/commons-lang-2/src/main/java/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSite.java +++ b/dd-java-agent/instrumentation/commons-lang-2/src/main/java/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSite.java @@ -6,7 +6,7 @@ import datadog.trace.api.iast.Propagation; import datadog.trace.api.iast.VulnerabilityMarks; import datadog.trace.api.iast.propagation.PropagationModule; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; @Propagation @CallSite(spi = IastCallSites.class) @@ -21,11 +21,11 @@ public class StringEscapeUtilsCallSite { @CallSite.After( "java.lang.String org.apache.commons.lang.StringEscapeUtils.escapeXml(java.lang.String)") public static String afterEscape( - @CallSite.Argument(0) @Nonnull final String input, @CallSite.Return final String result) { + @CallSite.Argument(0) @Nullable final String input, @CallSite.Return final String result) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { try { - module.taintIfInputIsTaintedWithMarks(result, input, VulnerabilityMarks.XSS_MARK); + module.taintIfTainted(result, input, false, VulnerabilityMarks.XSS_MARK); } catch (final Throwable e) { module.onUnexpectedException("afterEscape threw", e); } @@ -36,11 +36,11 @@ public static String afterEscape( @CallSite.After( "java.lang.String org.apache.commons.lang.StringEscapeUtils.escapeSql(java.lang.String)") public static String afterEscapeSQL( - @CallSite.Argument(0) @Nonnull final String input, @CallSite.Return final String result) { + @CallSite.Argument(0) @Nullable final String input, @CallSite.Return final String result) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { try { - module.taintIfInputIsTaintedWithMarks(result, input, VulnerabilityMarks.SQL_INJECTION_MARK); + module.taintIfTainted(result, input, false, VulnerabilityMarks.SQL_INJECTION_MARK); } catch (final Throwable e) { module.onUnexpectedException("afterEscapeSQL threw", e); } diff --git a/dd-java-agent/instrumentation/commons-lang-2/src/test/groovy/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSiteTest.groovy b/dd-java-agent/instrumentation/commons-lang-2/src/test/groovy/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSiteTest.groovy index 7a4d9457619..348aaa57503 100644 --- a/dd-java-agent/instrumentation/commons-lang-2/src/test/groovy/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSiteTest.groovy +++ b/dd-java-agent/instrumentation/commons-lang-2/src/test/groovy/datadog/trace/instrumentation/commonslang/StringEscapeUtilsCallSiteTest.groovy @@ -27,7 +27,7 @@ class StringEscapeUtilsCallSiteTest extends AgentTestRunner { then: result == expected - 1 * module.taintIfInputIsTaintedWithMarks(_ as String, args[0], mark) + 1 * module.taintIfTainted(_ as String, args[0], false, mark) 0 * _ where: diff --git a/dd-java-agent/instrumentation/commons-lang-3/src/main/java/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSite.java b/dd-java-agent/instrumentation/commons-lang-3/src/main/java/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSite.java index e151128f760..b52f57c57e9 100644 --- a/dd-java-agent/instrumentation/commons-lang-3/src/main/java/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSite.java +++ b/dd-java-agent/instrumentation/commons-lang-3/src/main/java/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSite.java @@ -6,7 +6,7 @@ import datadog.trace.api.iast.Propagation; import datadog.trace.api.iast.VulnerabilityMarks; import datadog.trace.api.iast.propagation.PropagationModule; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; @Propagation @CallSite(spi = IastCallSites.class) @@ -23,11 +23,11 @@ public class StringEscapeUtilsCallSite { @CallSite.After( "java.lang.String org.apache.commons.lang3.StringEscapeUtils.escapeEcmaScript(java.lang.String)") public static String afterEscape( - @CallSite.Argument(0) @Nonnull final String input, @CallSite.Return final String result) { + @CallSite.Argument(0) @Nullable final String input, @CallSite.Return final String result) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { try { - module.taintIfInputIsTaintedWithMarks(result, input, VulnerabilityMarks.XSS_MARK); + module.taintIfTainted(result, input, false, VulnerabilityMarks.XSS_MARK); } catch (final Throwable e) { module.onUnexpectedException("afterEscape threw", e); } @@ -38,11 +38,11 @@ public static String afterEscape( @CallSite.After( "java.lang.String org.apache.commons.lang3.StringEscapeUtils.escapeJson(java.lang.String)") public static String afterEscapeJson( - @CallSite.Argument(0) @Nonnull final String input, @CallSite.Return final String result) { + @CallSite.Argument(0) @Nullable final String input, @CallSite.Return final String result) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { try { - module.taintIfInputIsTainted(result, input); + module.taintIfTainted(result, input); } catch (final Throwable e) { module.onUnexpectedException("afterEscapeJson threw", e); } diff --git a/dd-java-agent/instrumentation/commons-lang-3/src/test/groovy/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSiteTest.groovy b/dd-java-agent/instrumentation/commons-lang-3/src/test/groovy/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSiteTest.groovy index dc7147fe6e4..18730b70612 100644 --- a/dd-java-agent/instrumentation/commons-lang-3/src/test/groovy/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSiteTest.groovy +++ b/dd-java-agent/instrumentation/commons-lang-3/src/test/groovy/datadog/trace/instrumentation/commonslang3/StringEscapeUtilsCallSiteTest.groovy @@ -25,7 +25,7 @@ class StringEscapeUtilsCallSiteTest extends AgentTestRunner { then: result == expected - 1 * module.taintIfInputIsTaintedWithMarks(_ as String, args[0], VulnerabilityMarks.XSS_MARK) + 1 * module.taintIfTainted(_ as String, args[0], false, VulnerabilityMarks.XSS_MARK) 0 * _ where: @@ -47,7 +47,7 @@ class StringEscapeUtilsCallSiteTest extends AgentTestRunner { then: result == expected - 1 * module.taintIfInputIsTainted(_ as String, args[0]) + 1 * module.taintIfTainted(_ as String, args[0]) 0 * _ where: diff --git a/dd-java-agent/instrumentation/commons-text/src/main/java/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSite.java b/dd-java-agent/instrumentation/commons-text/src/main/java/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSite.java index 68620efd078..ebd592fa241 100644 --- a/dd-java-agent/instrumentation/commons-text/src/main/java/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSite.java +++ b/dd-java-agent/instrumentation/commons-text/src/main/java/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSite.java @@ -6,7 +6,7 @@ import datadog.trace.api.iast.Propagation; import datadog.trace.api.iast.VulnerabilityMarks; import datadog.trace.api.iast.propagation.PropagationModule; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; @Propagation @CallSite(spi = IastCallSites.class) @@ -25,11 +25,11 @@ public class StringEscapeUtilsCallSite { @CallSite.After( "java.lang.String org.apache.commons.text.StringEscapeUtils.escapeXml11(java.lang.String)") public static String afterEscape( - @CallSite.Argument(0) @Nonnull final String input, @CallSite.Return final String result) { + @CallSite.Argument(0) @Nullable final String input, @CallSite.Return final String result) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { try { - module.taintIfInputIsTaintedWithMarks(result, input, VulnerabilityMarks.XSS_MARK); + module.taintIfTainted(result, input, false, VulnerabilityMarks.XSS_MARK); } catch (final Throwable e) { module.onUnexpectedException("afterEscape threw", e); } @@ -40,11 +40,11 @@ public static String afterEscape( @CallSite.After( "java.lang.String org.apache.commons.text.StringEscapeUtils.escapeJson(java.lang.String)") public static String afterEscapeJson( - @CallSite.Argument(0) @Nonnull final String input, @CallSite.Return final String result) { + @CallSite.Argument(0) @Nullable final String input, @CallSite.Return final String result) { final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { try { - module.taintIfInputIsTainted(result, input); + module.taintIfTainted(result, input); } catch (final Throwable e) { module.onUnexpectedException("afterEscapeJson threw", e); } diff --git a/dd-java-agent/instrumentation/commons-text/src/test/groovy/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSiteTest.groovy b/dd-java-agent/instrumentation/commons-text/src/test/groovy/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSiteTest.groovy index 12296ad58c2..440e31ea3be 100644 --- a/dd-java-agent/instrumentation/commons-text/src/test/groovy/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSiteTest.groovy +++ b/dd-java-agent/instrumentation/commons-text/src/test/groovy/datadog/trace/instrumentation/commonstext/StringEscapeUtilsCallSiteTest.groovy @@ -25,7 +25,7 @@ class StringEscapeUtilsCallSiteTest extends AgentTestRunner { then: result == expected - 1 * module.taintIfInputIsTaintedWithMarks(_ as String, args[0], VulnerabilityMarks.XSS_MARK) + 1 * module.taintIfTainted(_ as String, args[0], false, VulnerabilityMarks.XSS_MARK) 0 * _ where: @@ -48,7 +48,7 @@ class StringEscapeUtilsCallSiteTest extends AgentTestRunner { then: result == expected - 1 * module.taintIfInputIsTainted(_ as String, args[0]) + 1 * module.taintIfTainted(_ as String, args[0]) 0 * _ where: diff --git a/dd-java-agent/instrumentation/exception-profiling/src/main/java/datadog/exceptions/instrumentation/ThrowableInstrumentation.java b/dd-java-agent/instrumentation/exception-profiling/src/main/java/datadog/exceptions/instrumentation/ThrowableInstrumentation.java index 08a9b11997c..8f37f033e23 100644 --- a/dd-java-agent/instrumentation/exception-profiling/src/main/java/datadog/exceptions/instrumentation/ThrowableInstrumentation.java +++ b/dd-java-agent/instrumentation/exception-profiling/src/main/java/datadog/exceptions/instrumentation/ThrowableInstrumentation.java @@ -1,21 +1,15 @@ package datadog.exceptions.instrumentation; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.declaresField; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.isConstructor; import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.api.Platform; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; -/** Provides instrumentation of {@linkplain Throwable} constructor. */ +/** Provides instrumentation of {@linkplain Exception} and {@linkplain Error} constructors. */ @AutoService(Instrumenter.class) public final class ThrowableInstrumentation extends Instrumenter.Profiling - implements Instrumenter.ForBootstrap, - Instrumenter.ForSingleType, - Instrumenter.WithTypeStructure { + implements Instrumenter.ForBootstrap, Instrumenter.ForKnownTypes { public ThrowableInstrumentation() { super("throwables"); @@ -27,17 +21,14 @@ public boolean isEnabled() { } @Override - public String instrumentedType() { - return "java.lang.Throwable"; - } - - @Override - public ElementMatcher structureMatcher() { - return declaresField(named("stackTrace")); + public void adviceTransformations(AdviceTransformation transformation) { + transformation.applyAdvice(isConstructor(), packageName + ".ThrowableInstanceAdvice"); } @Override - public void adviceTransformations(AdviceTransformation transformation) { - transformation.applyAdvice(isConstructor(), packageName + ".ThrowableInstanceAdvice"); + public String[] knownMatchingTypes() { + return new String[] { + "java.lang.Exception", "java.lang.Error", "kotlin.Exception", "kotlin.Error" + }; } } diff --git a/dd-java-agent/instrumentation/exception-profiling/src/main/java11/datadog/exceptions/instrumentation/ThrowableInstanceAdvice.java b/dd-java-agent/instrumentation/exception-profiling/src/main/java11/datadog/exceptions/instrumentation/ThrowableInstanceAdvice.java index 32072e031ed..3c2623f5a48 100644 --- a/dd-java-agent/instrumentation/exception-profiling/src/main/java11/datadog/exceptions/instrumentation/ThrowableInstanceAdvice.java +++ b/dd-java-agent/instrumentation/exception-profiling/src/main/java11/datadog/exceptions/instrumentation/ThrowableInstanceAdvice.java @@ -11,12 +11,7 @@ public class ThrowableInstanceAdvice { @Advice.OnMethodExit(suppress = Throwable.class) - public static void onExit( - @Advice.This final Throwable t, - @Advice.FieldValue("stackTrace") StackTraceElement[] stackTrace) { - if (t.getClass().getName().endsWith(".ResourceLeakDetector$TraceRecord")) { - return; - } + public static void onExit(@Advice.This final Object t) { /* * This instrumentation handler is sensitive to any throwables thrown from its body - * it will go into infinite loop of trying to handle the new throwable instance and generating @@ -31,24 +26,23 @@ public static void onExit( } try { /* - * Exclude internal agent threads from exception profiling. + * We may get into a situation when this is called before exception sampling is active. */ - if (Config.get().isProfilingExcludeAgentThreads() - && AGENT_THREAD_GROUP.equals(Thread.currentThread().getThreadGroup())) { + if (!InstrumentationBasedProfiling.isJFRReady()) { return; } /* - * We may get into a situation when this is called before exception sampling is active. + * Exclude internal agent threads from exception profiling. */ - if (!InstrumentationBasedProfiling.isJFRReady()) { + if (Config.get().isProfilingExcludeAgentThreads() + && AGENT_THREAD_GROUP.equals(Thread.currentThread().getThreadGroup())) { return; } /* * JFR will assign the stacktrace depending on the place where the event is committed. * Therefore we need to commit the event here, right in the 'Exception' constructor */ - final ExceptionSampleEvent event = - ExceptionProfiling.getInstance().process(t, stackTrace == null ? 0 : stackTrace.length); + final ExceptionSampleEvent event = ExceptionProfiling.getInstance().process((Throwable) t); if (event != null && event.shouldCommit()) { event.commit(); } diff --git a/dd-java-agent/instrumentation/freemarker/src/main/java/datadog/trace/instrumentation/freemarker/StringUtilCallSite.java b/dd-java-agent/instrumentation/freemarker/src/main/java/datadog/trace/instrumentation/freemarker/StringUtilCallSite.java index 8ee9bc4ea2a..391339f528b 100644 --- a/dd-java-agent/instrumentation/freemarker/src/main/java/datadog/trace/instrumentation/freemarker/StringUtilCallSite.java +++ b/dd-java-agent/instrumentation/freemarker/src/main/java/datadog/trace/instrumentation/freemarker/StringUtilCallSite.java @@ -29,7 +29,7 @@ public static String afterEscape( final PropagationModule module = InstrumentationBridge.PROPAGATION; if (module != null) { try { - module.taintIfInputIsTaintedWithMarks(result, input, VulnerabilityMarks.XSS_MARK); + module.taintIfTainted(result, input, false, VulnerabilityMarks.XSS_MARK); } catch (final Throwable e) { module.onUnexpectedException("afterEscape threw", e); } diff --git a/dd-java-agent/instrumentation/freemarker/src/test/groovy/datadog/trace/instrumentation/freemarker/StringUtilCallSiteTest.groovy b/dd-java-agent/instrumentation/freemarker/src/test/groovy/datadog/trace/instrumentation/freemarker/StringUtilCallSiteTest.groovy index 51b0512a5e3..18d86d6aef5 100644 --- a/dd-java-agent/instrumentation/freemarker/src/test/groovy/datadog/trace/instrumentation/freemarker/StringUtilCallSiteTest.groovy +++ b/dd-java-agent/instrumentation/freemarker/src/test/groovy/datadog/trace/instrumentation/freemarker/StringUtilCallSiteTest.groovy @@ -23,17 +23,17 @@ class StringUtilCallSiteTest extends AgentTestRunner { then: result == expected - 1 * module.taintIfInputIsTaintedWithMarks(_ as String, args[0], VulnerabilityMarks.XSS_MARK) + 1 * module.taintIfTainted(_ as String, args[0], false, VulnerabilityMarks.XSS_MARK) 0 * _ where: - method | args | expected - 'HTMLEnc' | ['"escape this < '] | '<htmlTag>"escape this < </htmlTag>' - 'XMLEnc' | ['"escape this < '] | '<xmlTag>"escape this < </xmlTag>' - 'XHTMLEnc' | ['"escape this < '] | '<htmlTag>"escape this < </htmlTag>' - 'javaStringEnc' | ['