From a6b69d184e2eb1670c96b54c2c462b9ed4c53ef7 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 21 Nov 2024 15:56:02 -0800 Subject: [PATCH] support pytest-subtests plugin --- build/test-requirements.txt | 1 + .../.data/test_pytest_subtests_plugin.py | 8 +++++ .../expected_execution_test_output.py | 36 +++++++++++++++++++ .../tests/pytestadapter/test_execution.py | 28 +++++++++++++++ python_files/vscode_pytest/__init__.py | 13 ++++++- .../testController/common/resultResolver.ts | 4 +-- .../testing/testController/common/utils.ts | 10 ++++-- .../testing/testController/utils.unit.test.ts | 6 ++-- 8 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py diff --git a/build/test-requirements.txt b/build/test-requirements.txt index 8b0ea1636157..e92467468589 100644 --- a/build/test-requirements.txt +++ b/build/test-requirements.txt @@ -36,6 +36,7 @@ pytest-json # for pytest-describe related tests pytest-describe +pytest-subtests # for pytest-ruff related tests pytest-ruff diff --git a/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py b/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py new file mode 100644 index 000000000000..28615ce450f3 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +def test_a(subtests): + with subtests.test(msg="test_a"): + assert 1 == 1 + with subtests.test(msg="Second subtest"): + assert 2 == 1 diff --git a/python_files/tests/pytestadapter/expected_execution_test_output.py b/python_files/tests/pytestadapter/expected_execution_test_output.py index fa6743d0e112..a9a2180a344c 100644 --- a/python_files/tests/pytestadapter/expected_execution_test_output.py +++ b/python_files/tests/pytestadapter/expected_execution_test_output.py @@ -6,6 +6,7 @@ TEST_ADD_FUNCTION = "unittest_folder/test_add.py::TestAddFunction::" SUCCESS = "success" FAILURE = "failure" +SUBTEST_FAILURE = "subtest-failure" # This is the expected output for the unittest_folder execute tests # └── unittest_folder @@ -735,6 +736,41 @@ }, } +# This is the expected output for the test_pytest_subtests_plugin.py file. +# └── test_pytest_subtests_plugin.py +# └── test_a +# └── test_a [test_a]: subtest-success +# └── test_a [Second subtest]: subtest-failure +test_pytest_subtests_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py" +pytest_subtests_plugin_expected_execution_output = { + get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**", + test_pytest_subtests_plugin_path, + ): { + "test": get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**", + test_pytest_subtests_plugin_path, + ), + "outcome": "subtest-success", + "message": None, + "traceback": None, + "subtest": None, + }, + get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**", + test_pytest_subtests_plugin_path, + ): { + "test": get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**", + test_pytest_subtests_plugin_path, + ), + "outcome": SUBTEST_FAILURE, + "message": "ERROR MESSAGE", + "traceback": None, + "subtest": None, + }, +} + skip_test_fixture_path = TEST_DATA_PATH / "skip_test_fixture.py" skip_test_fixture_execution_expected_output = { get_absolute_test_id("skip_test_fixture.py::test_docker_client", skip_test_fixture_path): { diff --git a/python_files/tests/pytestadapter/test_execution.py b/python_files/tests/pytestadapter/test_execution.py index 95a66e0e7b87..c59d00d76298 100644 --- a/python_files/tests/pytestadapter/test_execution.py +++ b/python_files/tests/pytestadapter/test_execution.py @@ -232,6 +232,34 @@ def test_pytest_execution(test_ids, expected_const): assert actual_result_dict == expected_const +def test_pytest_subtest_plugin(): + test_subtest_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py" + + test_a_id = get_absolute_test_id( + "test_subtest_plugin_path::test_a", + test_subtest_plugin_path, + ) + args = [test_a_id] + expected_const = expected_execution_test_output.pytest_subtests_plugin_expected_execution_output + actual = runner(args) + assert actual + actual_list: List[Dict[str, Dict[str, Any]]] = actual + assert len(actual_list) == len(expected_const) + actual_result_dict = {} + if actual_list is not None: + for actual_item in actual_list: + assert all(item in actual_item for item in ("status", "cwd", "result")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + actual_result_dict.update(actual_item["result"]) + for key in actual_result_dict: + if actual_result_dict[key]["outcome"] == "subtest-failure": + actual_result_dict[key]["message"] = "ERROR MESSAGE" + if actual_result_dict[key]["traceback"] is not None: + actual_result_dict[key]["traceback"] = "TRACEBACK" + assert actual_result_dict == expected_const + + def test_symlink_run(): """Test to test pytest discovery with the command line arg --rootdir specified as a symlink path. diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 0ba5fd62221a..2a2b7a842a7b 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -271,8 +271,19 @@ def pytest_report_teststatus(report, config): # noqa: ARG001 node_path = map_id_to_path[report.nodeid] except KeyError: node_path = cwd - # Calculate the absolute test id and use this as the ID moving forward. + absolute_node_id = get_absolute_test_id(report.nodeid, node_path) + parent_test_name = report.nodeid.split("::")[-1] + if report.head_line and report.head_line != parent_test_name: + # add parent node to collected_tests_so_far to not double report + collected_tests_so_far.append(absolute_node_id) + # If the report has a head_line, then it is a pytest-subtest + # and we need to adjust the nodeid to reflect the subtest. + if report_value == "failure": + report_value = "subtest-failure" + elif report_value == "success": + report_value = "subtest-success" + absolute_node_id = absolute_node_id + "**{" + report.head_line + "}**" if absolute_node_id not in collected_tests_so_far: collected_tests_so_far.append(absolute_node_id) item_result = create_test_outcome( diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 80e57edbabd2..a0fbed408153 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -246,7 +246,7 @@ export class PythonResultResolver implements ITestResultResolver { } } else if (testItem.outcome === 'subtest-failure') { // split on [] or () based on how the subtest is setup. - const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp); + const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider); const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); const data = testItem; // find the subtest's parent test item @@ -288,7 +288,7 @@ export class PythonResultResolver implements ITestResultResolver { } } else if (testItem.outcome === 'subtest-success') { // split on [] or () based on how the subtest is setup. - const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp); + const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider); const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); // find the subtest's parent test item diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 68e10a2213d6..3120c235ecc3 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -18,6 +18,7 @@ import { import { Deferred, createDeferred } from '../../../common/utils/async'; import { createReaderPipe, generateRandomPipeName } from '../../../common/pipes/namedPipes'; import { EXTENSION_ROOT_DIR } from '../../../constants'; +import { TestProvider } from '../../types'; export function fixLogLinesNoTrailing(content: string): string { const lines = content.split(/\r?\n/g); @@ -272,10 +273,15 @@ export function createDiscoveryErrorPayload( * @param testName The full test name string. * @returns A tuple where the first item is the parent test name and the second item is the subtest section or `testName` if no subtest section exists. */ -export function splitTestNameWithRegex(testName: string): [string, string] { +export function splitTestNameWithRegex(testName: string, testProvider: TestProvider): [string, string] { // If a match is found, return the parent test name and the subtest (whichever was captured between parenthesis or square brackets). // Otherwise, return the entire testName for the parent and entire testName for the subtest. - const regex = /^(.*?) ([\[(].*[\])])$/; + let regex: RegExp; + if (testProvider === 'pytest') { + regex = /^(.*?)\*\*{(.*?)}\*\*$/; + } else { + regex = /^(.*?) ([\[(].*[\])])$/; + } const match = testName.match(regex); if (match) { return [match[1].trim(), match[2] || match[3] || testName]; diff --git a/src/test/testing/testController/utils.unit.test.ts b/src/test/testing/testController/utils.unit.test.ts index b871d18348e2..8f013da2ae8c 100644 --- a/src/test/testing/testController/utils.unit.test.ts +++ b/src/test/testing/testController/utils.unit.test.ts @@ -1,5 +1,5 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. +// // // Copyright (c) Microsoft Corporation. All rights reserved. +// // // Licensed under the MIT License. // import * as assert from 'assert'; // import { @@ -106,7 +106,7 @@ // assert.deepStrictEqual(rpcContent.remainingRawData, ''); // }); -// suite('Test Controller Utils: Other', () => { +// suite('Test Controller Utils: Unittest', () => { // interface TestCase { // name: string; // input: string;