diff --git a/build/test-requirements.txt b/build/test-requirements.txt index af19987bc8cb..e6fac80be1ff 100644 --- a/build/test-requirements.txt +++ b/build/test-requirements.txt @@ -36,3 +36,4 @@ pytest-json # for pytest-describe related tests pytest-describe +pytest-subtests diff --git a/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py b/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py new file mode 100644 index 000000000000..28615ce450f3 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/test_pytest_subtests_plugin.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +def test_a(subtests): + with subtests.test(msg="test_a"): + assert 1 == 1 + with subtests.test(msg="Second subtest"): + assert 2 == 1 diff --git a/python_files/tests/pytestadapter/expected_execution_test_output.py b/python_files/tests/pytestadapter/expected_execution_test_output.py index 8f378074343d..0ffd051989b9 100644 --- a/python_files/tests/pytestadapter/expected_execution_test_output.py +++ b/python_files/tests/pytestadapter/expected_execution_test_output.py @@ -6,6 +6,7 @@ TEST_ADD_FUNCTION = "unittest_folder/test_add.py::TestAddFunction::" SUCCESS = "success" FAILURE = "failure" +SUBTEST_FAILURE = "subtest-failure" # This is the expected output for the unittest_folder execute tests # └── unittest_folder @@ -734,3 +735,38 @@ "subtest": None, }, } + +# This is the expected output for the test_pytest_subtests_plugin.py file. +# └── test_pytest_subtests_plugin.py +# └── test_a +# └── test_a [test_a]: subtest-success +# └── test_a [Second subtest]: subtest-failure +test_pytest_subtests_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py" +pytest_subtests_plugin_expected_execution_output = { + get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**", + test_pytest_subtests_plugin_path, + ): { + "test": get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**", + test_pytest_subtests_plugin_path, + ), + "outcome": "subtest-success", + "message": None, + "traceback": None, + "subtest": None, + }, + get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**", + test_pytest_subtests_plugin_path, + ): { + "test": get_absolute_test_id( + "test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**", + test_pytest_subtests_plugin_path, + ), + "outcome": SUBTEST_FAILURE, + "message": "ERROR MESSAGE", + "traceback": None, + "subtest": None, + }, +} diff --git a/python_files/tests/pytestadapter/test_execution.py b/python_files/tests/pytestadapter/test_execution.py index 245b13cf5d46..d224b7eee111 100644 --- a/python_files/tests/pytestadapter/test_execution.py +++ b/python_files/tests/pytestadapter/test_execution.py @@ -227,6 +227,34 @@ def test_pytest_execution(test_ids, expected_const): assert actual_result_dict == expected_const +def test_pytest_subtest_plugin(): + test_subtest_plugin_path = TEST_DATA_PATH / "test_pytest_subtests_plugin.py" + + test_a_id = get_absolute_test_id( + "test_subtest_plugin_path::test_a", + test_subtest_plugin_path, + ) + args = [test_a_id] + expected_const = expected_execution_test_output.pytest_subtests_plugin_expected_execution_output + actual = runner(args) + assert actual + actual_list: List[Dict[str, Dict[str, Any]]] = actual + assert len(actual_list) == len(expected_const) + actual_result_dict = {} + if actual_list is not None: + for actual_item in actual_list: + assert all(item in actual_item for item in ("status", "cwd", "result")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + actual_result_dict.update(actual_item["result"]) + for key in actual_result_dict: + if actual_result_dict[key]["outcome"] == "subtest-failure": + actual_result_dict[key]["message"] = "ERROR MESSAGE" + if actual_result_dict[key]["traceback"] is not None: + actual_result_dict[key]["traceback"] = "TRACEBACK" + assert actual_result_dict == expected_const + + def test_symlink_run(): """Test to test pytest discovery with the command line arg --rootdir specified as a symlink path. diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 59a1b75e9688..f60db103b56d 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -278,8 +278,19 @@ def pytest_report_teststatus(report, config): # noqa: ARG001 node_path = map_id_to_path[report.nodeid] except KeyError: node_path = cwd - # Calculate the absolute test id and use this as the ID moving forward. + absolute_node_id = get_absolute_test_id(report.nodeid, node_path) + parent_test_name = report.nodeid.split("::")[-1] + if report.head_line and report.head_line != parent_test_name: + # add parent node to collected_tests_so_far to not double report + collected_tests_so_far.append(absolute_node_id) + # If the report has a head_line, then it is a pytest-subtest + # and we need to adjust the nodeid to reflect the subtest. + if report_value == "failure": + report_value = "subtest-failure" + elif report_value == "success": + report_value = "subtest-success" + absolute_node_id = absolute_node_id + "**{" + report.head_line + "}**" if absolute_node_id not in collected_tests_so_far: collected_tests_so_far.append(absolute_node_id) item_result = create_test_outcome( diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index d2b8fcaa24a5..4ea2bf9dedea 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -242,7 +242,7 @@ export class PythonResultResolver implements ITestResultResolver { } } else if (testItem.outcome === 'subtest-failure') { // split on [] or () based on how the subtest is setup. - const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp); + const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider); const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); const data = testItem; // find the subtest's parent test item @@ -280,7 +280,7 @@ export class PythonResultResolver implements ITestResultResolver { } } else if (testItem.outcome === 'subtest-success') { // split on [] or () based on how the subtest is setup. - const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp); + const [parentTestCaseId, subtestId] = splitTestNameWithRegex(keyTemp, this.testProvider); const parentTestItem = this.runIdToTestItem.get(parentTestCaseId); // find the subtest's parent test item diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 6c1492c2a9b7..e46da55ea14b 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -22,6 +22,7 @@ import { import { Deferred, createDeferred } from '../../../common/utils/async'; import { createReaderPipe, generateRandomPipeName } from '../../../common/pipes/namedPipes'; import { EXTENSION_ROOT_DIR } from '../../../constants'; +import { TestProvider } from '../../types'; export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); @@ -464,10 +465,15 @@ export function createDiscoveryErrorPayload( * @param testName The full test name string. * @returns A tuple where the first item is the parent test name and the second item is the subtest section or `testName` if no subtest section exists. */ -export function splitTestNameWithRegex(testName: string): [string, string] { +export function splitTestNameWithRegex(testName: string, testProvider: TestProvider): [string, string] { // If a match is found, return the parent test name and the subtest (whichever was captured between parenthesis or square brackets). // Otherwise, return the entire testName for the parent and entire testName for the subtest. - const regex = /^(.*?) ([\[(].*[\])])$/; + let regex: RegExp; + if (testProvider === 'pytest') { + regex = /^(.*?)\*\*{(.*?)}\*\*$/; + } else { + regex = /^(.*?) ([\[(].*[\])])$/; + } const match = testName.match(regex); if (match) { return [match[1].trim(), match[2] || match[3] || testName]; diff --git a/src/test/testing/testController/utils.unit.test.ts b/src/test/testing/testController/utils.unit.test.ts index dbf8b8249b9c..bba87825d92d 100644 --- a/src/test/testing/testController/utils.unit.test.ts +++ b/src/test/testing/testController/utils.unit.test.ts @@ -106,7 +106,7 @@ ${data}${secondPayload}`; assert.deepStrictEqual(rpcContent.remainingRawData, ''); }); - suite('Test Controller Utils: Other', () => { + suite('Test Controller Utils: Unittest', () => { interface TestCase { name: string; input: string; @@ -155,11 +155,55 @@ ${data}${secondPayload}`; testCases.forEach((testCase) => { test(`splitTestNameWithRegex: ${testCase.name}`, () => { - const splitResult = splitTestNameWithRegex(testCase.input); + const splitResult = splitTestNameWithRegex(testCase.input, 'unittest'); assert.deepStrictEqual(splitResult, [testCase.expectedParent, testCase.expectedSubtest]); }); }); }); + + suite('Test Controller Utils: Pytest', () => { + interface TestCase { + name: string; + input: string; + expectedParent: string; + expectedSubtest: string; + } + + const testCases: Array = [ + { + name: 'basic example', + input: 'test_pytest_subtests_plugin.py::test_a**{test_a [Second subtest]}**', + expectedParent: 'test_pytest_subtests_plugin.py::test_a', + expectedSubtest: 'test_a [Second subtest]', + }, + { + name: 'duplicate name', + input: 'test_pytest_subtests_plugin.py::test_a**{test_a [test_a]}**', + expectedParent: 'test_pytest_subtests_plugin.py::test_a', + expectedSubtest: 'test_a [test_a]', + }, + { + name: 'weird characters name', + input: 'test_pytest_subtests_plugin.py::L34Tc**{test_a111 [test_a]}**', + expectedParent: 'test_pytest_subtests_plugin.py::L34Tc', + expectedSubtest: 'test_a111 [test_a]', + }, + { + name: 'name with stars', + input: 'test_pytest_subtests_plugin.py::L34Tc**{test_a111**** [test_a]}**', + expectedParent: 'test_pytest_subtests_plugin.py::L34Tc', + expectedSubtest: 'test_a111**** [test_a]', + }, + ]; + + testCases.forEach((testCase) => { + test(`splitTestNameWithRegex: ${testCase.name}`, () => { + const splitResult = splitTestNameWithRegex(testCase.input, 'pytest'); + assert.deepStrictEqual(splitResult, [testCase.expectedParent, testCase.expectedSubtest]); + }); + }); + }); + suite('Test Controller Utils: Args Mapping', () => { suite('addValueIfKeyNotExist', () => { test('should add key-value pair if key does not exist', () => {