From 11d5d7bf1060ed980cb9f14b7f27b0ef504abeb0 Mon Sep 17 00:00:00 2001 From: Sean Truitt Date: Thu, 16 Mar 2023 17:03:17 -0400 Subject: [PATCH 1/4] Issue3549 adding test output to deploy task --- AUTHORS.rst | 1 + cumulusci/salesforce_api/metadata.py | 85 ++++++++++++++++++- cumulusci/tasks/salesforce/Deploy.py | 16 ++++ .../tasks/salesforce/tests/test_Deploy.py | 33 +++++++ .../salesforce/tests/test_DeployBundles.py | 2 + 5 files changed, 135 insertions(+), 2 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 1ac2d6bda3..70c1a2aab6 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -36,3 +36,4 @@ For example: * Ed Rivas (jerivas) * Gustavo Tandeciarz (dcinzona) +* John Truitt (jtruit) diff --git a/cumulusci/salesforce_api/metadata.py b/cumulusci/salesforce_api/metadata.py index 9c13e56656..51a7e7bc23 100644 --- a/cumulusci/salesforce_api/metadata.py +++ b/cumulusci/salesforce_api/metadata.py @@ -29,6 +29,7 @@ MetadataParseError, ) from cumulusci.utils import parse_api_datetime, zip_subfolder +from cumulusci.tasks.apex.testrunner import RunApexTests # If pyOpenSSL is installed, make sure it's not used for requests # (it's not needed in the verisons of Python we support) @@ -376,6 +377,7 @@ def __init__( self, task, package_zip, + options={}, purge_on_delete=None, api_version=None, check_only=False, @@ -390,7 +392,13 @@ def __init__( self.check_only = "true" if check_only else "false" self.test_level = test_level self.package_zip = package_zip - self.run_tests = run_tests or [] + self.options = options + # Set default values if nothing is passed + if self.options.get("junit_output") is None: + self.options["junit_output"] = "test_results.xml" + if self.options.get("json_output") is None: + self.options["json_output"] = "test_results.json" + self.run_tests = run_tests or [] def _set_purge_on_delete(self, purge_on_delete): if not purge_on_delete or purge_on_delete == "false": @@ -421,10 +429,74 @@ def _build_envelope_start(self): run_tests=run_tests, api_version=self.api_version, ) + + def _parse_elements(self, file): + # Create Array for Test Results + parsed_results = [] + + # Parse out test success to add to log + successElements = file.getElementsByTagName("successes") + failElements = file.getElementsByTagName("failures") + + for success in successElements: + # Get needed values from subelements + methodName = self._get_element_value(success, "methodName") + className = self._get_element_value(success, "name") + stats = {"duration": self._get_element_value(success, "time")} + + parsed_results.append( + { + "Children": None, + "ClassName": className, + "Method": methodName, + "Message": None, + "Outcome": None, + "StackTrace": None, + "Stats": stats, + "TestTimestamp": None, + } + ) + + for failures in failElements: + # Get needed values from subelements + methodName = self._get_element_value(failures, "methodName") + className = self._get_element_value(failures, "name") + stats = {"duration": self._get_element_value(failures, "time")} + # Parse out any failure text (from test failures in production + # deployments) and add to log + namespace = self._get_element_value(failures, "namespace") + stacktrace = self._get_element_value(failures, "stackTrace") + testMessage = self._get_element_value(failures, "message") + + message = ["Apex Test Failure: "] + if namespace: + message.append(f"from namespace {namespace}: ") + if stacktrace: + message.append(stacktrace) + # messages.append("".join(message)) + + parsed_results.append( + { + "Children": None, + "ClassName": className, + "Method": methodName, + "Message": testMessage, + "Outcome": "Fail", + "StackTrace": stacktrace, + "Stats": stats, + "TestTimestamp": None, + } + ) + + return parsed_results def _process_response(self, response): resp_xml = parseString(response.content) status = resp_xml.getElementsByTagName("status") + + # Create Array for Apex Test Results + test_results = [] + if status: status = status[0].firstChild.nodeValue else: @@ -436,6 +508,11 @@ def _process_response(self, response): # related to done if status in ["Succeeded", "SucceededPartial"]: self._set_status("Success", status) + + # Use existing function for apex tests to format and write output + test_results = self._parse_elements(resp_xml) + RunApexTests._write_output(self,test_results) + else: # If failed, parse out the problem text and raise appropriate exception messages = [] @@ -531,7 +608,11 @@ def _process_response(self, response): message.append(f"from namespace {namespace}: ") if stacktrace: message.append(stacktrace) - messages.append("".join(message)) + messages.append("".join(message)) + + # Use existing function for apex tests to format and write output + test_results = self._parse_elements(resp_xml) + RunApexTests._write_output(self,test_results) if messages: # Deploy failures due to a component failure should raise MetadataComponentFailure diff --git a/cumulusci/tasks/salesforce/Deploy.py b/cumulusci/tasks/salesforce/Deploy.py index f84ae3ae24..395628d031 100644 --- a/cumulusci/tasks/salesforce/Deploy.py +++ b/cumulusci/tasks/salesforce/Deploy.py @@ -55,6 +55,12 @@ class Deploy(BaseSalesforceMetadataApiTask): "transforms": { "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms." }, + "junit_output": { + "description": "XML test result output filename. Defaults to test_results.xml." + }, + "json_output": { + "description": "JSON test result output filename. Defaults to test_results.json." + }, } namespaces = {"sf": "http://soap.sforce.com/2006/04/metadata"} @@ -64,6 +70,12 @@ class Deploy(BaseSalesforceMetadataApiTask): def _init_options(self, kwargs): super(Deploy, self)._init_options(kwargs) + # Set default values if nothing is passed + if self.options.get("junit_output") is None: + self.options["junit_output"] = "test_results.xml" + if self.options.get("json_output") is None: + self.options["json_output"] = "test_results.json" + self.check_only = process_bool_arg(self.options.get("check_only", False)) self.test_level = self.options.get("test_level") if self.test_level and self.test_level not in [ @@ -109,10 +121,14 @@ def _get_api(self, path=None): else: self.logger.warning("Deployment package is empty; skipping deployment.") return + + options = {"junit_output": self.options["junit_output"], + "json_output": self.options["json_output"]} return self.api_class( self, package_zip, + options, purge_on_delete=False, check_only=self.check_only, test_level=self.test_level, diff --git a/cumulusci/tasks/salesforce/tests/test_Deploy.py b/cumulusci/tasks/salesforce/tests/test_Deploy.py index a08211f117..b5f89476d6 100644 --- a/cumulusci/tasks/salesforce/tests/test_Deploy.py +++ b/cumulusci/tasks/salesforce/tests/test_Deploy.py @@ -196,6 +196,39 @@ def test_init_options__bad_transforms(self): assert "transform spec is not valid" in str(e) + def test_init_options__output_default(self): + d = create_task( + Deploy, + { + "path": "src", + }, + ) + + assert d.options["junit_output"] == "test_results.xml" + assert d.options["json_output"] == "test_results.json" + + def test_init_options__JUNIT_output(self): + d = create_task( + Deploy, + { + "path": "src", + "junit_output": "TEST.xml", + }, + ) + + assert d.options["junit_output"] == "TEST.xml" + + def test_init_options__JSON_output(self): + d = create_task( + Deploy, + { + "path": "src", + "json_output": "TEST.json", + }, + ) + + assert d.options["json_output"] == "TEST.json" + def test_freeze_sets_kind(self): task = create_task( Deploy, diff --git a/cumulusci/tasks/salesforce/tests/test_DeployBundles.py b/cumulusci/tasks/salesforce/tests/test_DeployBundles.py index 6ccd8eb129..911fc16e2b 100644 --- a/cumulusci/tasks/salesforce/tests/test_DeployBundles.py +++ b/cumulusci/tasks/salesforce/tests/test_DeployBundles.py @@ -58,6 +58,8 @@ def test_freeze(self): "github": "https://github.com/TestOwner/TestRepo", "subfolder": "unpackaged/test", "namespace_inject": None, + 'junit_output': 'test_results.xml', + 'json_output': 'test_results.json', } ] }, From c0461e7d87db3604534444f476583a89c205325a Mon Sep 17 00:00:00 2001 From: Sean Truitt Date: Tue, 6 Jun 2023 12:34:39 -0400 Subject: [PATCH 2/4] Issue3549 removing left behind commented out line updating with Dev --- cumulusci/salesforce_api/metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cumulusci/salesforce_api/metadata.py b/cumulusci/salesforce_api/metadata.py index 51a7e7bc23..61077fb356 100644 --- a/cumulusci/salesforce_api/metadata.py +++ b/cumulusci/salesforce_api/metadata.py @@ -473,7 +473,6 @@ def _parse_elements(self, file): message.append(f"from namespace {namespace}: ") if stacktrace: message.append(stacktrace) - # messages.append("".join(message)) parsed_results.append( { From 746ca246516de928445e01234c684011c8a75be1 Mon Sep 17 00:00:00 2001 From: John Truitt Date: Mon, 28 Aug 2023 14:04:31 -0400 Subject: [PATCH 3/4] Issue3549 Adding test to validate contents of file Fixing comment --- cumulusci/salesforce_api/metadata.py | 2 +- .../salesforce_api/tests/test_metadata.py | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/cumulusci/salesforce_api/metadata.py b/cumulusci/salesforce_api/metadata.py index 61077fb356..856d69705d 100644 --- a/cumulusci/salesforce_api/metadata.py +++ b/cumulusci/salesforce_api/metadata.py @@ -434,7 +434,7 @@ def _parse_elements(self, file): # Create Array for Test Results parsed_results = [] - # Parse out test success to add to log + # Parse out test results to add to log successElements = file.getElementsByTagName("successes") failElements = file.getElementsByTagName("failures") diff --git a/cumulusci/salesforce_api/tests/test_metadata.py b/cumulusci/salesforce_api/tests/test_metadata.py index 7e578a34bb..b25a960a70 100644 --- a/cumulusci/salesforce_api/tests/test_metadata.py +++ b/cumulusci/salesforce_api/tests/test_metadata.py @@ -44,6 +44,7 @@ ) from cumulusci.tests.util import DummyOrgConfig, create_project_config +from cumulusci.utils import temporary_dir, touch class DummyPackageZipBuilder(BasePackageZipBuilder): def _populate_zip(self): @@ -775,6 +776,45 @@ def test_process_response_failure_but_no_message(self): api._process_response(response) assert response.text == str(e.value) + def test_process_response_validate_test_output(self): + json_file = '.\\test_results.json' + xml_file = '.\\test_results.xml' + task = self._create_task() + api = self._create_instance(task) + response = Response() + response.status_code = 200 + response.raw = io.BytesIO( + deploy_result_failure.format( + details=""" + + test + stack + + +""" + ).encode() + ) + + with temporary_dir(): + with pytest.raises(ApexTestException): + api._process_response(response) + + if touch(json_file): + expected = '"Outcome": "Fail"' + + assert expected in json_file + + else: + assert False + + if touch(xml_file): + expected = '' + + assert expected in xml_file + + else: + assert False + def test_get_action(self): task = self._create_task() api = self._create_instance(task) From 5ad1dc24f99db376eac633d852cb34263efe5a35 Mon Sep 17 00:00:00 2001 From: John Truitt Date: Tue, 26 Sep 2023 08:07:33 -0400 Subject: [PATCH 4/4] Issue3549 adding new parameter --- cumulusci/tasks/salesforce/tests/test_Deploy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cumulusci/tasks/salesforce/tests/test_Deploy.py b/cumulusci/tasks/salesforce/tests/test_Deploy.py index c72b6e207c..8b650e09ea 100644 --- a/cumulusci/tasks/salesforce/tests/test_Deploy.py +++ b/cumulusci/tasks/salesforce/tests/test_Deploy.py @@ -223,7 +223,7 @@ def test_init_options__bad_transforms(self, rest_deploy): assert "transform spec is not valid" in str(e) @pytest.mark.parametrize("rest_deploy", [True, False]) - def test_init_options__output_default(self): + def test_init_options__output_default(self, rest_deploy): d = create_task( Deploy, { @@ -236,7 +236,7 @@ def test_init_options__output_default(self): assert d.options["json_output"] == "test_results.json" @pytest.mark.parametrize("rest_deploy", [True, False]) - def test_init_options__JUNIT_output(self): + def test_init_options__JUNIT_output(self, rest_deploy): d = create_task( Deploy, { @@ -249,7 +249,7 @@ def test_init_options__JUNIT_output(self): assert d.options["junit_output"] == "TEST.xml" @pytest.mark.parametrize("rest_deploy", [True, False]) - def test_init_options__JSON_output(self): + def test_init_options__JSON_output(self, rest_deploy): d = create_task( Deploy, {