Issue3549 adding test output to deploy task

AUTHORS.rst

@@ -38,3 +38,4 @@ For example:
 * Gustavo Tandeciarz (dcinzona)
 * Chandler Anderson (zenibako)
 * Ben French (BenjaminFrench)
+* John Truitt (jtruit)

cumulusci/salesforce_api/metadata.py

@@ -29,6 +29,7 @@
     MetadataParseError,
 )
 from cumulusci.utils import parse_api_datetime, zip_subfolder
+from cumulusci.tasks.apex.testrunner import RunApexTests
 
 # If pyOpenSSL is installed, make sure it's not used for requests
 # (it's not needed in the verisons of Python we support)
@@ -376,6 +377,7 @@ def __init__(
         self,
         task,
         package_zip,
+        options={},
         purge_on_delete=None,
         api_version=None,
         check_only=False,
@@ -390,7 +392,13 @@ def __init__(
         self.check_only = "true" if check_only else "false"
         self.test_level = test_level
         self.package_zip = package_zip
-        self.run_tests = run_tests or []
+        self.options = options   
+        # Set default values if nothing is passed
+        if self.options.get("junit_output") is None:        
+            self.options["junit_output"] = "test_results.xml"
+        if self.options.get("json_output") is None:        
+            self.options["json_output"] = "test_results.json"
+        self.run_tests = run_tests or []     
 
     def _set_purge_on_delete(self, purge_on_delete):
         if not purge_on_delete or purge_on_delete == "false":
@@ -421,10 +429,73 @@ def _build_envelope_start(self):
             run_tests=run_tests,
             api_version=self.api_version,
         )
+
+    def _parse_elements(self, file):
+        # Create Array for Test Results
+        parsed_results = []
+
+        # Parse out test results to add to log
+        successElements = file.getElementsByTagName("successes")
+        failElements = file.getElementsByTagName("failures")
+
+        for success in successElements:
+            # Get needed values from subelements
+            methodName = self._get_element_value(success, "methodName")
+            className = self._get_element_value(success, "name")
+            stats = {"duration": self._get_element_value(success, "time")}
+
+            parsed_results.append(
+                {
+                    "Children": None,
+                    "ClassName": className,
+                    "Method": methodName,
+                    "Message": None,
+                    "Outcome": None,
+                    "StackTrace": None,
+                    "Stats": stats,
+                    "TestTimestamp": None,
+                }
+            )
+
+        for failures in failElements:
+            # Get needed values from subelements
+            methodName = self._get_element_value(failures, "methodName")
+            className = self._get_element_value(failures, "name")
+            stats = {"duration": self._get_element_value(failures, "time")}
+            # Parse out any failure text (from test failures in production
+            # deployments) and add to log
+            namespace = self._get_element_value(failures, "namespace")
+            stacktrace = self._get_element_value(failures, "stackTrace")
+            testMessage = self._get_element_value(failures, "message")
+
+            message = ["Apex Test Failure: "]
+            if namespace:
+                message.append(f"from namespace {namespace}: ")
+            if stacktrace:
+                message.append(stacktrace)
+
+            parsed_results.append(
+                {
+                    "Children": None,
+                    "ClassName": className,
+                    "Method": methodName,
+                    "Message": testMessage,
+                    "Outcome": "Fail",
+                    "StackTrace": stacktrace,
+                    "Stats": stats,
+                    "TestTimestamp": None,
+                }
+            )   
+
+        return parsed_results
 
     def _process_response(self, response):
         resp_xml = parseString(response.content)
         status = resp_xml.getElementsByTagName("status")
+
+        # Create Array for Apex Test Results
+        test_results = []
+
         if status:
             status = status[0].firstChild.nodeValue
         else:
@@ -436,6 +507,11 @@ def _process_response(self, response):
         # related to done
         if status in ["Succeeded", "SucceededPartial"]:
             self._set_status("Success", status)
+
+            # Use existing function for apex tests to format and write output
+            test_results = self._parse_elements(resp_xml)
+            RunApexTests._write_output(self,test_results)
+
         else:
             # If failed, parse out the problem text and raise appropriate exception
             messages = []
@@ -531,7 +607,11 @@ def _process_response(self, response):
                     message.append(f"from namespace {namespace}: ")
                 if stacktrace:
                     message.append(stacktrace)
-                messages.append("".join(message))
+                messages.append("".join(message))            
+
+            # Use existing function for apex tests to format and write output
+            test_results = self._parse_elements(resp_xml)
+            RunApexTests._write_output(self,test_results)
 
             if messages:
                 # Deploy failures due to a component failure should raise MetadataComponentFailure

cumulusci/salesforce_api/tests/test_metadata.py

@@ -47,6 +47,7 @@
 )
 from cumulusci.tests.util import DummyOrgConfig, create_project_config
 
+from cumulusci.utils import temporary_dir, touch
 
 class DummyPackageZipBuilder(BasePackageZipBuilder):
     def _populate_zip(self):
@@ -778,6 +779,45 @@ def test_process_response_failure_but_no_message(self):
             api._process_response(response)
         assert response.text == str(e.value)
 
+    def test_process_response_validate_test_output(self):
+        json_file = '.\\test_results.json'
+        xml_file = '.\\test_results.xml'
+        task = self._create_task()
+        api = self._create_instance(task)
+        response = Response()
+        response.status_code = 200
+        response.raw = io.BytesIO(
+            deploy_result_failure.format(
+                details="""<runTestResult>
+  <failures>
+    <namespace>test</namespace>
+    <stackTrace>stack</stackTrace>
+  </failures>
+</runTestResult>
+"""
+            ).encode()
+        )
+
+        with temporary_dir():
+            with pytest.raises(ApexTestException):
+                api._process_response(response)
+
+                if touch(json_file):
+                    expected = '"Outcome": "Fail"'
+
+                    assert expected in json_file
+
+                else:
+                    assert False
+
+                if touch(xml_file):
+                    expected = '<failure type="failed" >'
+
+                    assert expected in xml_file
+
+                else:
+                    assert False
+
     def test_get_action(self):
         task = self._create_task()
         api = self._create_instance(task)

cumulusci/tasks/salesforce/Deploy.py

@@ -65,6 +65,12 @@ class Deploy(BaseSalesforceMetadataApiTask):
             "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms."
         },
         "rest_deploy": {"description": "If True, deploy metadata using REST API"},
+        "junit_output": {
+            "description": "XML test result output filename. Defaults to test_results.xml."
+        },
+        "json_output": {
+            "description": "JSON test result output filename. Defaults to test_results.json."
+        },
     }
 
     namespaces = {"sf": "http://soap.sforce.com/2006/04/metadata"}
@@ -74,6 +80,12 @@ class Deploy(BaseSalesforceMetadataApiTask):
     def _init_options(self, kwargs):
         super(Deploy, self)._init_options(kwargs)
 
+        # Set default values if nothing is passed
+        if self.options.get("junit_output") is None:        
+            self.options["junit_output"] = "test_results.xml"
+        if self.options.get("json_output") is None:        
+            self.options["json_output"] = "test_results.json"
+
         self.check_only = process_bool_arg(self.options.get("check_only", False))
         self.test_level = self.options.get("test_level")
         if self.test_level and self.test_level not in [
@@ -139,6 +151,9 @@ def _get_api(self, path=None):
         else:
             self.logger.warning("Deployment package is empty; skipping deployment.")
             return
+
+        options = {"junit_output": self.options["junit_output"],
+                   "json_output": self.options["json_output"]}
 
         # If rest_deploy param is set, update api_class to be RestDeploy
         if self.rest_deploy:
@@ -147,6 +162,7 @@ def _get_api(self, path=None):
         return self.api_class(
             self,
             package_zip,
+            options,
             purge_on_delete=False,
             check_only=self.check_only,
             test_level=self.test_level,

cumulusci/tasks/salesforce/tests/test_Deploy.py

@@ -401,6 +401,45 @@ def test_init_options__bad_transforms(self, rest_deploy):
             )
 
             assert "transform spec is not valid" in str(e)
+
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_init_options__output_default(self, rest_deploy):
+        d = create_task(
+            Deploy,
+            {
+                "path": "src",          
+                "rest_deploy": rest_deploy,    
+            },
+        )
+
+        assert d.options["junit_output"] == "test_results.xml"
+        assert d.options["json_output"] == "test_results.json"
+
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_init_options__JUNIT_output(self, rest_deploy):
+        d = create_task(
+            Deploy,
+            {
+                "path": "src",
+                "junit_output": "TEST.xml",     
+                "rest_deploy": rest_deploy,           
+            },
+        )
+
+        assert d.options["junit_output"] == "TEST.xml"
+
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_init_options__JSON_output(self, rest_deploy):
+        d = create_task(
+            Deploy,
+            {
+                "path": "src",
+                "json_output": "TEST.json",   
+                "rest_deploy": rest_deploy,             
+            },
+        )
+
+        assert d.options["json_output"] == "TEST.json" 
 
     @pytest.mark.parametrize("rest_deploy", [True, False])
     def test_freeze_sets_kind(self, rest_deploy):

cumulusci/tasks/salesforce/tests/test_DeployBundles.py

@@ -61,6 +61,8 @@ def test_freeze(self):
                                     "subfolder": "unpackaged/test",
                                     "collision_check": False,
                                     "namespace_inject": None,
+                                    'junit_output': 'test_results.xml', 
+                                    'json_output': 'test_results.json',
                                 }
                             ]
                         },