diff --git a/CHANGELOG.md b/CHANGELOG.md index 1970cc3..93a5091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Fixed +- Backgrounds finish with `FAILED` status, by @HardNorth ## [5.0.2] ### Changed diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java b/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java index f90b20f..749684f 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java @@ -23,6 +23,7 @@ import com.epam.reportportal.service.Launch; import com.epam.reportportal.service.ReportPortal; import com.epam.reportportal.utils.MemoizingSupplier; +import com.epam.reportportal.utils.StatusEvaluation; import com.epam.ta.reportportal.ws.model.FinishExecutionRQ; import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; import com.epam.ta.reportportal.ws.model.StartTestItemRQ; @@ -59,6 +60,7 @@ public class ReportPortalHook implements RuntimeHook { private final BlockingConcurrentHashMap> featureIdMap = new BlockingConcurrentHashMap<>(); private final Map> scenarioIdMap = new ConcurrentHashMap<>(); private final Map> backgroundIdMap = new ConcurrentHashMap<>(); + private final Map backgroundStatusMap = new ConcurrentHashMap<>(); private final Map> stepIdMap = new ConcurrentHashMap<>(); private final Map, Date> stepStartTimeMap = new ConcurrentHashMap<>(); private volatile Thread shutDownHook; @@ -228,6 +230,7 @@ protected StartTestItemRQ buildStartBackgroundRq(@Nonnull Step step, @Nonnull Sc * * @param step Karate's Step object instance * @param sr Karate's ScenarioRuntime object instance + * @return item ID Future */ public Maybe startBackground(@Nonnull Step step, @Nonnull ScenarioRuntime sr) { return backgroundIdMap.computeIfAbsent(sr.scenario.getUniqueId(), k -> { @@ -239,27 +242,28 @@ public Maybe startBackground(@Nonnull Step step, @Nonnull ScenarioRuntim /** * Build ReportPortal request for finish Background event. * - * @param step Karate's Step object instance - * @param sr Karate's ScenarioRuntime object instance + * @param stepResult Karate's StepResult class instance + * @param sr Karate's ScenarioRuntime object instance * @return request to ReportPortal */ @Nonnull @SuppressWarnings("unused") - protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable Step step, @Nonnull ScenarioRuntime sr) { - return buildFinishTestItemRq(Calendar.getInstance().getTime(), null); + protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable StepResult stepResult, @Nonnull ScenarioRuntime sr) { + return buildFinishTestItemRq(Calendar.getInstance().getTime(), backgroundStatusMap.remove(sr.scenario.getUniqueId())); } /** * Finish sending Scenario data to ReportPortal. * - * @param step Karate's Step object instance - * @param sr Karate's ScenarioRuntime object instance + * @param stepResult Karate's StepResult class instance + * @param sr Karate's ScenarioRuntime object instance */ - public void finishBackground(@Nullable Step step, @Nonnull ScenarioRuntime sr) { - Maybe backgroundId = backgroundIdMap.remove(sr.scenario.getUniqueId()); + public void finishBackground(@Nullable StepResult stepResult, @Nonnull ScenarioRuntime sr) { + String uniqueId = sr.scenario.getUniqueId(); + Maybe backgroundId = backgroundIdMap.remove(uniqueId); if (backgroundId != null) { - FinishTestItemRQ finishRq = buildFinishBackgroundRq(step, sr); + FinishTestItemRQ finishRq = buildFinishBackgroundRq(stepResult, sr); //noinspection ReactiveStreamsUnusedPublisher launch.get().finishTestItem(backgroundId, finishRq); } @@ -272,10 +276,10 @@ public void afterScenario(ScenarioRuntime sr) { LOGGER.error("ERROR: Trying to finish unspecified scenario."); } + finishBackground(null, sr); FinishTestItemRQ rq = buildFinishScenarioRq(sr); //noinspection ReactiveStreamsUnusedPublisher launch.get().finishTestItem(scenarioId, rq); - finishBackground(null, sr); } /** @@ -331,8 +335,6 @@ public boolean beforeStep(Step step, ScenarioRuntime sr) { Maybe backgroundId = null; if (background) { backgroundId = startBackground(step, sr); - } else { - finishBackground(step, sr); } StartTestItemRQ stepRq = buildStartStepRq(step, sr); @@ -383,8 +385,21 @@ protected FinishTestItemRQ buildFinishStepRq(@Nonnull StepResult stepResult, @No return buildFinishTestItemRq(Calendar.getInstance().getTime(), getStepStatus(stepResult.getResult().getStatus())); } + private void saveBackgroundStatus(@Nonnull StepResult stepResult, @Nonnull ScenarioRuntime sr) { + backgroundStatusMap.put(sr.scenario.getUniqueId(), + StatusEvaluation.evaluateStatus(backgroundStatusMap.get(sr.scenario.getUniqueId()), + getStepStatus(stepResult.getResult().getStatus()) + ) + ); + } + @Override public void afterStep(StepResult stepResult, ScenarioRuntime sr) { + boolean background = stepResult.getStep().isBackground(); + if (!background) { + finishBackground(stepResult, sr); + } + sendStepResults(stepResult, sr); Maybe stepId = stepIdMap.get(sr.scenario.getUniqueId()); if (stepId == null) { @@ -393,6 +408,9 @@ public void afterStep(StepResult stepResult, ScenarioRuntime sr) { } FinishTestItemRQ rq = buildFinishStepRq(stepResult, sr); + if (background) { + saveBackgroundStatus(stepResult, sr); + } //noinspection ReactiveStreamsUnusedPublisher launch.get().finishTestItem(stepId, rq); } diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java b/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java index 90d68e8..eb6bb13 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java @@ -22,6 +22,7 @@ import com.epam.reportportal.service.Launch; import com.epam.reportportal.service.ReportPortal; import com.epam.reportportal.utils.MemoizingSupplier; +import com.epam.reportportal.utils.StatusEvaluation; import com.epam.ta.reportportal.ws.model.FinishExecutionRQ; import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; import com.epam.ta.reportportal.ws.model.StartTestItemRQ; @@ -52,6 +53,7 @@ public class ReportPortalPublisher { private final Map> scenarioIdMap = new HashMap<>(); private final Map, Long> stepStartTimeMap = new HashMap<>(); private Maybe backgroundId; + private ItemStatus backgroundStatus; private Maybe stepId; private Thread shutDownHook; @@ -270,7 +272,7 @@ public void startBackground(@Nonnull StepResult stepResult, @Nonnull ScenarioRes @Nonnull @SuppressWarnings("unused") protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable StepResult stepResult, @Nonnull ScenarioResult scenarioResult) { - return buildFinishTestItemRq(Calendar.getInstance().getTime(), null); + return buildFinishTestItemRq(Calendar.getInstance().getTime(), backgroundStatus); } @@ -281,12 +283,13 @@ protected FinishTestItemRQ buildFinishBackgroundRq(@Nullable StepResult stepResu * @param scenarioResult scenario result */ public void finishBackground(@Nullable StepResult stepResult, @Nonnull ScenarioResult scenarioResult) { - ofNullable(backgroundId).ifPresent(id -> { + Maybe myBackgroundId = backgroundId; + backgroundId = null; + ofNullable(myBackgroundId).ifPresent(id -> { FinishTestItemRQ finishRq = buildFinishBackgroundRq(stepResult, scenarioResult); //noinspection ReactiveStreamsUnusedPublisher launch.get().finishTestItem(id, finishRq); }); - backgroundId = null; } /** @@ -332,14 +335,17 @@ protected StartTestItemRQ buildStartStepRq(@Nonnull StepResult stepResult, @Nonn */ public void startStep(StepResult stepResult, ScenarioResult scenarioResult) { Step step = stepResult.getStep(); - if (step.isBackground()) { + boolean background = step.isBackground(); + if (background) { startBackground(stepResult, scenarioResult); - } else { - finishBackground(stepResult, scenarioResult); } + StartTestItemRQ stepRq = buildStartStepRq(stepResult, scenarioResult); stepId = launch.get() - .startTestItem(backgroundId != null ? backgroundId : scenarioIdMap.get(scenarioResult.getScenario().getName()), stepRq); + .startTestItem( + background && backgroundId != null ? backgroundId : scenarioIdMap.get(scenarioResult.getScenario().getName()), + stepRq + ); stepStartTimeMap.put(stepId, stepRq.getStartTime().getTime()); ofNullable(stepRq.getParameters()).filter(params -> !params.isEmpty()) .ifPresent(params -> sendLog(stepId, String.format(PARAMETERS_PATTERN, formatParametersAsTable(params)), LogLevel.INFO)); @@ -363,6 +369,11 @@ protected FinishTestItemRQ buildFinishStepRq(@Nonnull StepResult stepResult, @No return buildFinishTestItemRq(Calendar.getInstance().getTime(), getStepStatus(stepResult.getResult().getStatus())); } + @SuppressWarnings("unused") + private void saveBackgroundStatus(@Nonnull StepResult stepResult, @Nonnull ScenarioResult scenarioResult) { + backgroundStatus = StatusEvaluation.evaluateStatus(backgroundStatus, getStepStatus(stepResult.getResult().getStatus())); + } + /** * Finish sending Step data to ReportPortal. * @@ -370,12 +381,21 @@ protected FinishTestItemRQ buildFinishStepRq(@Nonnull StepResult stepResult, @No * @param scenarioResult Karate's ScenarioResult class instance */ public void finishStep(StepResult stepResult, ScenarioResult scenarioResult) { + Step step = stepResult.getStep(); + boolean background = step.isBackground(); + if (!background) { + finishBackground(stepResult, scenarioResult); + } + if (stepId == null) { LOGGER.error("ERROR: Trying to finish unspecified step."); return; } FinishTestItemRQ rq = buildFinishStepRq(stepResult, scenarioResult); + if (background) { + saveBackgroundStatus(stepResult, scenarioResult); + } //noinspection ReactiveStreamsUnusedPublisher launch.get().finishTestItem(stepId, rq); } diff --git a/src/test/java/com/epam/reportportal/karate/status/OneExampleWithBackgroundFailedTest.java b/src/test/java/com/epam/reportportal/karate/status/OneExampleWithBackgroundFailedTest.java new file mode 100644 index 0000000..5687698 --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/status/OneExampleWithBackgroundFailedTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.status; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.ItemStatus; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.intuit.karate.Results; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.Mockito.*; + +public class OneExampleWithBackgroundFailedTest { + private static final String TEST_FEATURE = "classpath:feature/examples_one_failed_with_background.feature"; + private final String featureId = CommonUtils.namedId("feature_"); + private final List exampleIds = Stream.generate(() -> CommonUtils.namedId("example_")).limit(2).collect(Collectors.toList()); + private final List>> stepIds = exampleIds.stream() + .map(e -> Pair.of(e, Stream.generate(() -> CommonUtils.namedId("step_")).limit(3).collect(Collectors.toList()))) + .collect(Collectors.toList()); + private final List> nestedSteps = + Arrays.asList( + Pair.of(stepIds.get(0).getValue().get(0), CommonUtils.namedId("nested_step_")), + Pair.of(stepIds.get(1).getValue().get(0), CommonUtils.namedId("nested_step_")) + ); + + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, null, featureId, stepIds); + mockNestedSteps(client, nestedSteps); + mockBatchLogging(client); + } + + private static void verifyStatus(List rqs, ItemStatus... statuses) { + for (int i = 0; i < rqs.size(); i++) { + ItemStatus statusTest = i >= statuses.length ? statuses[statuses.length - 1] : statuses[i]; + assertThat( + "Failed verifying request number: " + (i + 1), + rqs.get(i).getStatus(), + allOf(notNullValue(), equalTo(statusTest.name())) + ); + } + } + + @ParameterizedTest + @ValueSource(booleans = { true, false }) + public void test_simple_one_step_failed(boolean report) { + Results results; + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + assertThat(results.getFailCount(), equalTo(1)); + + ArgumentCaptor featureCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(featureId), featureCaptor.capture()); + ArgumentCaptor firstExampleCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(exampleIds.get(0)), firstExampleCaptor.capture()); + ArgumentCaptor secondExampleCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(exampleIds.get(1)), secondExampleCaptor.capture()); + ArgumentCaptor firstExampleFirstStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(stepIds.get(0).getValue().get(0)), firstExampleFirstStepCaptor.capture()); + ArgumentCaptor firstExampleSecondStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(stepIds.get(0).getValue().get(1)), firstExampleSecondStepCaptor.capture()); + ArgumentCaptor firstExampleThirdStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(stepIds.get(0).getValue().get(2)), firstExampleThirdStepCaptor.capture()); + ArgumentCaptor secondExampleFirstStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(stepIds.get(1).getValue().get(0)), secondExampleFirstStepCaptor.capture()); + ArgumentCaptor secondExampleSecondStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(stepIds.get(1).getValue().get(1)), secondExampleSecondStepCaptor.capture()); + ArgumentCaptor secondExampleThirdStepCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(stepIds.get(1).getValue().get(2)), secondExampleThirdStepCaptor.capture()); + + FinishTestItemRQ featureRq = featureCaptor.getValue(); + assertThat(featureRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.FAILED.name()))); + + verifyStatus(Arrays.asList(firstExampleCaptor.getValue(), secondExampleCaptor.getValue()), ItemStatus.PASSED, ItemStatus.FAILED); + + List steps = Arrays.asList( + firstExampleFirstStepCaptor.getValue(), + firstExampleSecondStepCaptor.getValue(), + firstExampleThirdStepCaptor.getValue(), + secondExampleFirstStepCaptor.getValue(), + secondExampleSecondStepCaptor.getValue(), + secondExampleThirdStepCaptor.getValue() + ); + + verifyStatus(steps, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.PASSED, ItemStatus.FAILED); + } +} diff --git a/src/test/resources/feature/examples_one_failed_with_background.feature b/src/test/resources/feature/examples_one_failed_with_background.feature new file mode 100644 index 0000000..d5a292d --- /dev/null +++ b/src/test/resources/feature/examples_one_failed_with_background.feature @@ -0,0 +1,13 @@ +Feature: math tests with examples + + Background: Set varb + Given def varb = 2 + + Scenario Outline: Verify different maths + Given def mathResult = vara + varb + Then assert mathResult == result + + Examples: + | vara! | result! | + | 2 | 4 | + | 1 | 4 |