feat: add error handling without invoking the google test test environment

.vscode/launch.json

@@ -9,18 +9,21 @@
             "description": "Enter the tag to run",
             "type": "pickString",
             "options": [
+                "@smoke",
                 "@rule1 or @smoke",
                 "@keyword-asterisk",
-                "@result:UNDEFINED"
+                "@result:UNDEFINED",
+                "@result:FAILED",
+                "@fail_feature"
             ]
         },
         {
             "id": "features",
             "description": "Enter the tag to run",
             "type": "pickString",
             "options": [
-                "cucumber-cpp-example/features",
-                "test/features"
+                "cucumber_cpp/example/features",
+                "cucumber_cpp/acceptance_test/features"
             ]
         }
     ],
@@ -36,9 +39,10 @@
                 "${input:features}",
                 "--report",
                 "console",
-                "--com",
-                "COMx",
-                "--nordic",
+                "junit-xml",
+                // "--com",
+                // "COMx",
+                // "--nordic",
                 "--tag",
                 "${input:tag}"
             ],

.vscode/settings.json

     "projectKey": "philips-software_amp-cucumber-cpp-runner" 
 "cmake.useCMakePresets": "always", 
 "cucumberautocomplete.steps": [ 
     "cucumber_cpp/example/steps/*.cpp", 
     "cucumber_cpp/acceptance_test/steps/*.cpp" 
 ], 
 "cucumberautocomplete.gherkinDefinitionPart": "(GIVEN|WHEN|THEN|STEP)\\(", 
 "cucumberautocomplete.customParameters": [ 
     { 
         "parameter": "R\"(", 
         "value": "\"" 
     }, 
     { 
         "parameter": ")\"", 
         "value": "\"" 
     } 
 ], 
 "C/C++ Include Guard.Macro Type": "Filepath", 
 "C/C++ Include Guard.Path Depth": 1, 
 "C/C++ Include Guard.Remove Extension": false, 
 "C/C++ Include Guard.Comment Style": "None", 
 "C/C++ Include Guard.Path Skip": 0, 
 "testMate.cpp.test.executables": "${workspaceFolder}/.build/**/*{test,Test,TEST}*", 
 "sonarlint.connectedMode.project": { 
     "connectionId": "philips-software", 
     "projectKey": "philips-software_amp-cucumber-cpp-runner" 
@@ -1,8 +1,8 @@
 {
     "cmake.useCMakePresets": "always",
     "cucumberautocomplete.steps": [
-        "cucumber-cpp-example/steps/*.cpp",
-        "test/steps/*.cpp"
+        "cucumber_cpp/example/steps/*.cpp",
+        "cucumber_cpp/acceptance_test/steps/*.cpp"
     ],
     "cucumberautocomplete.gherkinDefinitionPart": "(GIVEN|WHEN|THEN|STEP)\\(",
     "cucumberautocomplete.customParameters": [
@@ -20,5 +20,9 @@
     "C/C++ Include Guard.Remove Extension": false,
     "C/C++ Include Guard.Comment Style": "None",
     "C/C++ Include Guard.Path Skip": 0,
-    "testMate.cpp.test.executables": "${workspaceFolder}/.build/**/*{test,Test,TEST}*"
+    "testMate.cpp.test.executables": "${workspaceFolder}/.build/**/*{test,Test,TEST}*",
+    "sonarlint.connectedMode.project": {
+        "connectionId": "philips-software",
+        "projectKey": "philips-software_amp-cucumber-cpp-runner"
+    }
 }

.vscode/tasks.json

@@ -6,7 +6,7 @@
         {
             "label": "bats",
             "type": "shell",
-            "command": "bats --formatter junit cucumber_cpp/integration_test/test.bats | tee test-report.xml",
+            "command": "bats --formatter junit cucumber_cpp/acceptance_test/test.bats | tee test-report.xml",
             "problemMatcher": []
         }
     ]

CMakeLists.txt

@@ -27,7 +27,7 @@ if (CCR_BUILD_TESTS)
     ccr_enable_testing()
 endif()
 
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED On)
 set(CMAKE_CXX_EXTENSIONS Off)
 

cucumber_cpp/acceptance_test/features/test_fail_feature.feature

@@ -0,0 +1,15 @@
+@fail_feature
+Feature: Simple feature file
+    Background:
+        Given a background step
+
+    Scenario: A failing scenario
+        Given a given step
+        When a when step
+        Then an assertion is raised
+        Then a then step
+
+    Scenario: An OK scenario
+        Given a given step
+        When a when step
+        Then a then step

cucumber_cpp/acceptance_test/features/test_hooks.feature

@@ -1,23 +1,37 @@
 @bats
 Feature: Test scenario and step hook bindings
 
-  Background:
-    Given a background step
-
-  @scenariohook @stephook
-  Scenario: Run Scenario and Step hooks
+  @fail_scenariohook_before
+  Scenario: Run failing Scenario hooks before
     Given a given step
-    When a when step
-    Then a then step
 
-  @scenariohook
-  Scenario: Run only Scenario hooks
+  @fail_scenariohook_after
+  Scenario: Run failing Scenario hooks after
     Given a given step
-    When a when step
-    Then a then step
 
-  @stephook
-  Scenario: Run only Step hooks
+  @throw_scenariohook
+  Scenario: Run throwing Scenario hooks
     Given a given step
-    When a when step
-    Then a then step
+
+  Rule: These scenarios have a background
+
+    Background:
+      Given a background step
+
+    @scenariohook @stephook
+    Scenario: Run Scenario and Step hooks
+      Given a given step
+      When a when step
+      Then a then step
+
+    @scenariohook
+    Scenario: Run only Scenario hooks
+      Given a given step
+      When a when step
+      Then a then step
+
+    @stephook
+    Scenario: Run only Step hooks
+      Given a given step
+      When a when step
+      Then a then step

cucumber_cpp/acceptance_test/hooks/Hooks.cpp

@@ -1,4 +1,5 @@
 #include "cucumber_cpp/library/Hooks.hpp"
+#include "gmock/gmock.h"
 #include <iostream>
 
 HOOK_BEFORE_ALL()
@@ -30,3 +31,18 @@ HOOK_AFTER_STEP("@stephook and @bats")
 {
     std::cout << "HOOK_AFTER_STEP\n";
 }
+
+HOOK_BEFORE_SCENARIO("@fail_scenariohook_before")
+{
+    ASSERT_THAT(false, testing::IsTrue());
+}
+
+HOOK_AFTER_SCENARIO("@fail_scenariohook_after")
+{
+    ASSERT_THAT(false, testing::IsTrue());
+}
+
+HOOK_BEFORE_SCENARIO("@throw_scenariohook")
+{
+    throw "error";
+}

cucumber_cpp/acceptance_test/steps/Steps.cpp

@@ -44,3 +44,8 @@ THEN("two expectations are raised")
     EXPECT_THAT(0, testing::Eq(1));
     EXPECT_THAT(1, testing::Eq(0));
 }
+
+WHEN("I print {string} with value {int}", (const std::string& str, std::int32_t value))
+{
+    std::cout << "print: " << str << " with value " << value;
+}

cucumber_cpp/acceptance_test/test.bats

@@ -6,7 +6,7 @@ setup() {
 }
 
 teardown() {
-    rm -rf out.json out.xml
+    rm -rf ./out/ ./test-report.xml
 }
 
 @test "Successful test" {
@@ -148,3 +148,31 @@ teardown() {
     run .build/Host/cucumber_cpp/acceptance_test/Debug/cucumber_cpp.acceptance_test run --feature cucumber_cpp/acceptance_test/features --tag "@keyword-asterisk" --report console
     assert_failure
 }
+
+@test "Test passing scenario after failed scenario reports feature as failed" {
+    run .build/Host/cucumber_cpp/acceptance_test/Debug/cucumber_cpp.acceptance_test run --feature cucumber_cpp/acceptance_test/features  --tag "@fail_feature" --report console
+    assert_failure
+    assert_output --partial "tests   : 1/2 passed"
+}
+
+@test "Test failing hook before results in error" {
+    run .build/Host/cucumber_cpp/acceptance_test/Debug/cucumber_cpp.acceptance_test run --feature cucumber_cpp/acceptance_test/features  --tag "@fail_scenariohook_before" --report console
+    assert_failure
+    assert_output --partial "skipped Given a given step"
+    assert_output --partial "tests   : 0/1 passed"
+}
+
+@test "Test failing hook after results in error" {
+    run .build/Host/cucumber_cpp/acceptance_test/Debug/cucumber_cpp.acceptance_test run --feature cucumber_cpp/acceptance_test/features  --tag "@fail_scenariohook_after" --report console
+    assert_failure
+    assert_output --partial "Given a given step"
+    assert_output --partial "-> done"
+    assert_output --partial "tests   : 0/1 passed"
+}
+
+@test "Test throwing hook results in error" {
+    run .build/Host/cucumber_cpp/acceptance_test/Debug/cucumber_cpp.acceptance_test run --feature cucumber_cpp/acceptance_test/features  --tag "@throw_scenariohook" --report console
+    assert_failure
+    assert_output --partial "skipped Given a given step"
+    assert_output --partial "tests   : 0/1 passed"
+}

cucumber_cpp/example/steps/Steps.cpp

@@ -1,5 +1,6 @@
 #include "cucumber_cpp/library/Steps.hpp"
 #include "cucumber_cpp/library/Context.hpp"
+#include "cucumber_cpp/library/engine/TestFailureHandler.hpp"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <cstdint>
@@ -8,7 +9,7 @@
 
 GIVEN(R"(a background step)")
 {
-    std::cout << "\nthis is a background step\n";
+    std::cout << "\nthis is a background step";
 }
 
 GIVEN(R"(a simple data table)")

cucumber_cpp/library/Application.cpp

@@ -3,7 +3,9 @@
 #include "cucumber_cpp/library/engine/ContextManager.hpp"
 #include "cucumber_cpp/library/engine/FeatureFactory.hpp"
 #include "cucumber_cpp/library/engine/FeatureInfo.hpp"
+#include "cucumber_cpp/library/engine/HookExecutor.hpp"
 #include "cucumber_cpp/library/engine/Result.hpp"
+#include "cucumber_cpp/library/engine/TestExecution.hpp"
 #include "cucumber_cpp/library/engine/TestRunner.hpp"
 #include "cucumber_cpp/library/report/JunitReport.hpp"
 #include "cucumber_cpp/library/report/Report.hpp"
@@ -86,7 +88,7 @@ namespace cucumber_cpp
 
     ResultStatus& ResultStatus::operator=(Result result)
     {
-        if ((resultStatus == Result::undefined || resultStatus == Result::success) && result != Result::undefined)
+        if ((resultStatus == Result::undefined || resultStatus == Result::passed) && result != Result::undefined)
             resultStatus = result;
 
         return *this;
@@ -99,12 +101,14 @@ namespace cucumber_cpp
 
     bool ResultStatus::IsSuccess() const
     {
-        return resultStatus == Result::success;
+        return resultStatus == Result::passed;
     }
 
     Application::Application(std::shared_ptr<ContextStorageFactory> contextStorageFactory)
-        : reportHandlerValidator{ reporters }
-        , contextManager{ std::move(contextStorageFactory) }
+        : contextManager{ std::move(contextStorageFactory) }
+        , reporters{ contextManager }
+        , reportHandlerValidator{ reporters }
+
     {
         gherkin.include_source(false);
         gherkin.include_ast(true);
@@ -156,7 +160,7 @@ namespace cucumber_cpp
 
     Context& Application::ProgramContext()
     {
-        return contextManager.CurrentContext();
+        return contextManager.ProgramContext();
     }
 
     const Application::Options& Application::CliOptions() const
@@ -175,11 +179,18 @@ namespace cucumber_cpp
             reporters.Use(selectedReporter);
 
         auto tagExpression = Join(options.tags, " ");
+        library::engine::HookExecutorImpl hookExecution{ contextManager };
+        library::engine::TestExecutionImpl testExecution{ contextManager, reporters, hookExecution, [this]() -> const library::engine::TestExecution::Policy&
+            {
+                if (options.dryrun)
+                    return library::engine::dryRunPolicy;
+                else
+                    return library::engine::executeRunPolicy;
+            }() };
 
-        if (options.dryrun)
-            engine::TestRunner::Run(contextManager, GetFeatureTree(tagExpression), reporters, engine::dryRun);
-        else
-            engine::TestRunner::Run(contextManager, GetFeatureTree(tagExpression), reporters, engine::runTest);
+        engine::TestRunnerImpl testRunner{ testExecution };
+
+        testRunner.Run(GetFeatureTree(tagExpression));
 
         std::cout << '\n'
                   << std::flush;
@@ -201,7 +212,7 @@ namespace cucumber_cpp
 
     int Application::GetExitCode() const
     {
-        if (contextManager.CurrentContext().ExecutionStatus() == engine::Result::passed)
+        if (contextManager.ProgramContext().ExecutionStatus() == engine::Result::passed)
             return 0;
         else
             return 1;

cucumber_cpp/library/Application.hpp

@@ -25,7 +25,7 @@ namespace cucumber_cpp
 
     struct ResultStatus
     {
-        using Result = cucumber_cpp::report::ReportHandler::Result;
+        using Result = cucumber_cpp::engine::Result;
 
         ResultStatus& operator=(Result result);
         explicit operator Result() const;
@@ -66,19 +66,20 @@ namespace cucumber_cpp
         void DryRunFeatures();
         void RunFeatures();
         [[nodiscard]] std::vector<std::unique_ptr<engine::FeatureInfo>> GetFeatureTree(std::string_view tagExpression);
-        [[nodiscard]] report::ReportHandler::Result RunFeature(const std::filesystem::path& path, std::string_view tagExpression, report::ReportHandlerV2& reportHandler);
+        [[nodiscard]] cucumber_cpp::engine::Result RunFeature(const std::filesystem::path& path, std::string_view tagExpression, report::ReportHandlerV2& reportHandler);
 
         Options options;
         CLI::App cli;
         CLI::App* runCommand;
 
-        report::ReportForwarder reporters;
+        engine::ContextManager contextManager;
+
+        report::ReportForwarderImpl reporters;
         ReportHandlerValidator reportHandlerValidator;
 
         cucumber::gherkin::app gherkin;
 
         engine::FeatureTreeFactory featureTreeFactory{};
-        engine::ContextManager contextManager;
     };
 }
 

cucumber_cpp/library/BodyMacro.hpp

@@ -3,6 +3,7 @@
 
 #include "cucumber_cpp/library/Body.hpp"
 #include "cucumber_cpp/library/engine/StringTo.hpp"
+#include "cucumber_cpp/library/engine/TestFailureHandler.hpp"
 #include <cstddef>
 #include <functional>
 
@@ -51,4 +52,9 @@
     const std::size_t BODY_STRUCT::ID = cucumber_cpp::registration<BODY_STRUCT>(matcher, type);                  \
     void BODY_STRUCT::ExecuteWithArgs targs
 
+#undef GTEST_MESSAGE_AT_
+#define GTEST_MESSAGE_AT_(file, line, message, result_type)                                 \
+    cucumber_cpp::library::engine::CucumberAssertHelper(result_type, file, line, message) = \
+        ::testing::Message()
+
 #endif

cucumber_cpp/library/CMakeLists.txt

@@ -11,11 +11,7 @@ target_sources(cucumber_cpp.library PRIVATE
     HookRegistry.cpp
     HookRegistry.hpp
     Hooks.hpp
-    HookScopes.cpp
-    HookScopes.hpp
     InternalError.hpp
-    OnTestPartResultEventListener.cpp
-    OnTestPartResultEventListener.hpp
     Rtrim.cpp
     Rtrim.hpp
     Steps.hpp
@@ -38,6 +34,7 @@ target_link_libraries(cucumber_cpp.library PUBLIC
     cucumber_gherkin_lib
     cucumber_cpp.library.report
     cucumber_cpp.library.engine
+    cucumber_cpp.library.util
     CLI11
 )
 
@@ -51,6 +48,7 @@ target_compile_options(cucumber_cpp.library
 
 add_subdirectory(engine)
 add_subdirectory(report)
+add_subdirectory(util)
 
 if (CCR_BUILD_TESTS)
     add_subdirectory(test)