Fix NPE in @setup calls when tracing Karate (#6146)

.github/CODEOWNERS

@@ -19,6 +19,7 @@ dd-java-agent/instrumentation/cucumber/    @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/jacoco/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/junit-4.10/  @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/junit-5.3/   @DataDog/ci-app-libraries-java
+dd-java-agent/instrumentation/karate/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/testng/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/gradle/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/maven-3.2.1/ @DataDog/ci-app-libraries-java

dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java

@@ -20,14 +20,17 @@
 import datadog.trace.bootstrap.instrumentation.api.Tags;
 import java.util.Collection;
 
-// FIXME nikita: do not trace Karate tests in JUnit 4 / JUnit 5 instrumentations
 public class KarateTracingHook implements RuntimeHook {
 
   private static final String FRAMEWORK_NAME = "karate";
   private static final String FRAMEWORK_VERSION = FileUtils.KARATE_VERSION;
+  private static final String KARATE_STEP_SPAN_NAME = "karate.step";
 
   @Override
   public boolean beforeFeature(FeatureRuntime fr) {
+    if (skipTracking(fr)) {
+      return true;
+    }
     Feature feature = KarateUtils.getFeature(fr);
     Suite suite = fr.suite;
     TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
@@ -42,6 +45,9 @@ public boolean beforeFeature(FeatureRuntime fr) {
 
   @Override
   public void afterFeature(FeatureRuntime fr) {
+    if (skipTracking(fr)) {
+      return;
+    }
     String featureName = KarateUtils.getFeature(fr).getNameForReport();
     FeatureResult result = fr.result;
     if (result.isFailed()) {
@@ -55,6 +61,9 @@ public void afterFeature(FeatureRuntime fr) {
 
   @Override
   public boolean beforeScenario(ScenarioRuntime sr) {
+    if (skipTracking(sr)) {
+      return true;
+    }
     Scenario scenario = sr.scenario;
     Feature feature = scenario.getFeature();
 
@@ -70,7 +79,7 @@ public boolean beforeScenario(ScenarioRuntime sr) {
         TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore(
             featureName,
             scenarioName,
-            scenario.getRefId(),
+            sr,
             FRAMEWORK_NAME,
             FRAMEWORK_VERSION,
             parameters,
@@ -86,7 +95,7 @@ public boolean beforeScenario(ScenarioRuntime sr) {
     TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart(
         featureName,
         scenarioName,
-        scenario.getRefId(),
+        sr,
         FRAMEWORK_NAME,
         FRAMEWORK_VERSION,
         parameters,
@@ -99,6 +108,9 @@ public boolean beforeScenario(ScenarioRuntime sr) {
 
   @Override
   public void afterScenario(ScenarioRuntime sr) {
+    if (skipTracking(sr)) {
+      return;
+    }
     Scenario scenario = sr.scenario;
     Feature feature = scenario.getFeature();
 
@@ -111,25 +123,23 @@ public void afterScenario(ScenarioRuntime sr) {
           featureName,
           null,
           scenarioName,
-          scenario.getRefId(),
+          sr,
           KarateUtils.getParameters(scenario),
           result.getError());
     } else if (result.getStepResults().isEmpty()) {
       TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(
-          featureName,
-          null,
-          scenarioName,
-          scenario.getRefId(),
-          KarateUtils.getParameters(scenario),
-          null);
+          featureName, null, scenarioName, sr, KarateUtils.getParameters(scenario), null);
     }
     TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(
-        featureName, null, scenarioName, scenario.getRefId(), KarateUtils.getParameters(scenario));
+        featureName, null, scenarioName, sr, KarateUtils.getParameters(scenario));
   }
 
   @Override
   public boolean beforeStep(Step step, ScenarioRuntime sr) {
-    AgentSpan span = AgentTracer.startSpan("karate", "karate.step");
+    if (skipTracking(step)) {
+      return true;
+    }
+    AgentSpan span = AgentTracer.startSpan("karate", KARATE_STEP_SPAN_NAME);
     AgentScope scope = AgentTracer.activateSpan(span);
     String stepName = step.getPrefix() + " " + step.getText();
     span.setResourceName(stepName);
@@ -143,6 +153,10 @@ public boolean beforeStep(Step step, ScenarioRuntime sr) {
 
   @Override
   public void afterStep(StepResult result, ScenarioRuntime sr) {
+    if (skipTracking(result.getStep())) {
+      return;
+    }
+
     AgentSpan span = AgentTracer.activeSpan();
     if (span == null) {
       return;
@@ -155,4 +169,22 @@ public void afterStep(StepResult result, ScenarioRuntime sr) {
 
     span.finish();
   }
+
+  private static boolean skipTracking(FeatureRuntime fr) {
+    // do not track nested feature calls
+    return !fr.caller.isNone();
+  }
+
+  private static boolean skipTracking(ScenarioRuntime sr) {
+    // do not track nested scenario calls and setup scenarios
+    return !sr.caller.isNone() || sr.tags.getTagKeys().contains("setup");
+  }
+
+  private static boolean skipTracking(Step step) {
+    // do not track steps that are not children of a tracked scenario or another tracked step
+    AgentSpan activeSpan = AgentTracer.activeSpan();
+    return activeSpan == null
+        || (!KARATE_STEP_SPAN_NAME.contentEquals(activeSpan.getSpanName())
+            && !Tags.SPAN_KIND_TEST.contentEquals(activeSpan.getSpanType()));
+  }
 }

dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy

@@ -13,6 +13,7 @@ import org.example.TestFailedKarate
 import org.example.TestParameterizedKarate
 import org.example.TestSucceedKarate
 import org.example.TestUnskippableKarate
+import org.example.TestWithSetupKarate
 import org.junit.jupiter.api.Assumptions
 import org.junit.jupiter.engine.JupiterTestEngine
 import org.junit.platform.engine.DiscoverySelector
@@ -50,6 +51,34 @@ class KarateTest extends CiVisibilityTest {
     })
   }
 
+  def "test scenarios with setup generate spans"() {
+    given:
+    Assumptions.assumeTrue(isSetupTagSupported(FileUtils.KARATE_VERSION),"Current Karate version is ${FileUtils.KARATE_VERSION}, while @setup tags are supported starting with 1.3.0")
+
+    when:
+    runTests(TestWithSetupKarate)
+
+    then:
+    ListWriterAssert.assertTraces(TEST_WRITER, 3, false, SORT_TRACES_BY_DESC_SIZE_THEN_BY_NAMES, {
+      long testSessionId
+      long testModuleId
+      long testSuiteId
+      trace(3, true) {
+        testSessionId = testSessionSpan(it, 1, CIConstants.TEST_PASS)
+        testModuleId = testModuleSpan(it, 0, testSessionId, CIConstants.TEST_PASS)
+        testSuiteId = testSuiteSpan(it, 2, testSessionId, testModuleId, "[org/example/test_with_setup] test with setup", CIConstants.TEST_PASS, null, null, false, [], false)
+      }
+      trace(2, true) {
+        long testId = testSpan(it, 1, testSessionId, testModuleId, testSuiteId, "[org/example/test_with_setup] test with setup", "first scenario", null, CIConstants.TEST_PASS, [(Tags.TEST_PARAMETERS): '{"foo":"bar"}'], null, false, ['withSetup'], false, false)
+        karateStepSpan(it, 0, testId, "* print foo", 15, 15)
+      }
+      trace(2, true) {
+        long testId = testSpan(it, 1, testSessionId, testModuleId, testSuiteId, "[org/example/test_with_setup] test with setup", "second scenario", null, CIConstants.TEST_PASS, [(Tags.TEST_PARAMETERS): '{"foo":"bar"}'], null, false, ['withSetupOnce'], false, false)
+        karateStepSpan(it, 0, testId, "* print foo", 21, 21)
+      }
+    })
+  }
+
   def "test parameterized generates spans"() {
     setup:
     runTests(TestParameterizedKarate)
@@ -305,4 +334,8 @@ class KarateTest extends CiVisibilityTest {
     // earlier Karate version contain a bug that does not allow skipping scenarios
     frameworkVersion >= "1.2.0"
   }
+
+  boolean isSetupTagSupported(String frameworkVersion) {
+    frameworkVersion >= "1.3.0"
+  }
 }

dd-java-agent/instrumentation/karate/src/test/java/org/example/TestWithSetupKarate.java

@@ -0,0 +1,11 @@
+package org.example;
+
+import com.intuit.karate.junit5.Karate;
+
+public class TestWithSetupKarate {
+
+  @Karate.Test
+  public Karate testSucceed() {
+    return Karate.run("classpath:org/example/test_with_setup.feature");
+  }
+}

dd-java-agent/instrumentation/karate/src/test/java/org/example/karate-config.js

@@ -0,0 +1,2 @@
+function fn() {
+}

dd-java-agent/instrumentation/karate/src/test/java/org/example/test_with_setup.feature

@@ -0,0 +1,27 @@
+Feature: test with setup
+
+  @setup
+  Scenario: setup scenario
+    * call read('@setupStep')
+    * def data =
+    """
+    [
+     {foo: "bar"}
+    ]
+    """
+
+  @withSetup
+  Scenario Outline: first scenario
+    * print foo
+    Examples:
+      | karate.setup().data |
+
+  @withSetupOnce
+  Scenario Outline: second scenario
+    * print foo
+    Examples:
+      | karate.setupOnce().data |
+
+  @ignore @setupStep
+  Scenario: setup step
+    * print 'test'