[WIP] Implement serve flag for planemo test

docs/_writing_test_and_serve.rst

@@ -19,5 +19,21 @@ Now we can open Galaxy with the ``serve`` (or just ``s``).
 Open up http://127.0.0.1:9090 in a web browser to view your new tool.
 
 Serve and test can be passed various command line arguments such as
-``--galaxy_root`` to specify a Galaxy instance to use (by default 
-planemo will download and manage a instance just for planemo).
+``--galaxy_root`` to specify the root of a development Galaxy directory
+to use (by default planemo will download and manage a instance just for planemo).
+
+Finally, Planemo also allows us to combine the features of ``test`` and
+``serve`` using the following command:
+
+::
+
+    $ planemo t --serve
+    ...
+    All 1 test(s) executed passed.
+    ...
+
+This command runs the specified tests, just like ``planemo test``, but
+instead of shutting down the temporary Galaxy server afterwards, it
+keeps it running, just like ``planemo serve``. You can open
+http://127.0.0.1:9090 in a web browser and view the datasets generated
+by the tests there. Each test is contained in a separate Galaxy history.

planemo/commands/cmd_serve.py

@@ -22,9 +22,9 @@ def cli(ctx, uris, **kwds):
     tool.
 
     planemo will search parent directories to see if any is a Galaxy instance
-    - but one can pick the Galaxy instance to use with the ``--galaxy_root``
-    option or force planemo to download a disposable instance with the
-    ``--install_galaxy`` flag.
+    - but one can pick the root of a development Galaxy directory to use with
+    the ``--galaxy_root`` option or force planemo to download a disposable
+    Galaxy with the ``--install_galaxy`` flag.
 
     ``planemo`` will run the Galaxy instance in an existing virtualenv if one
     exists in a ``.venv`` directory in the specified ``--galaxy_root``.

planemo/commands/cmd_test.py

@@ -1,4 +1,6 @@
 """Module describing the planemo ``test`` command."""
+from contextlib import ExitStack
+
 import click
 
 from planemo import options
@@ -32,7 +34,8 @@
          "instances to limit generated traffic.",
     default="0",
 )
-@options.galaxy_target_options()
+@options.serve_option()
+@options.galaxy_run_options()
 @options.galaxy_config_options()
 @options.test_options()
 @options.engine_options()
@@ -65,7 +68,11 @@ def cli(ctx, uris, **kwds):
     against that same Galaxy root - but this may not be bullet proof yet so
     please careful and do not try this against production Galaxy instances.
     """
-    with temp_directory(dir=ctx.planemo_directory) as temp_path:
+    with ExitStack() as stack:
+        if not kwds["serve"]:
+            temp_path = stack.enter_context(temp_directory(dir=ctx.planemo_directory))
+        else:
+            temp_path = None
         # Create temp dir(s) outside of temp, docker can't mount $TEMPDIR on OSX
         runnables = for_runnable_identifiers(ctx, uris, kwds, temp_path=temp_path)
 

planemo/engine/galaxy.py

@@ -77,13 +77,17 @@ def _register_job_data(job_data):
                         # and capture the output information somehow.
                         interactor.VERBOSE_GALAXY_ERRORS = True
 
+                    # on an external Galaxy it makes sense to keep test histories
+                    no_history_cleanup = config._kwds.get('engine') == 'external_galaxy'
+
                     interactor.verify_tool(
                         tool_id,
                         galaxy_interactor,
                         test_index=test_index,
                         tool_version=tool_version,
                         register_job_data=_register_job_data,
                         quiet=not verbose,
+                        no_history_cleanup=no_history_cleanup,
                     )
                 except Exception:
                     pass

planemo/engine/test.py

@@ -1,25 +1,51 @@
+import os
+
 from planemo.engine import (
     engine_context,
 )
 from planemo.galaxy import galaxy_config
+from planemo.galaxy import galaxy_serve
+from planemo.galaxy.api import (
+    DEFAULT_ADMIN_API_KEY
+)
 from planemo.galaxy.config import _find_test_data
+from planemo.galaxy.ephemeris_sleep import sleep
 from planemo.galaxy.test import (
     handle_reports_and_summary,
     run_in_config,
 )
 from planemo.runnable import (
+    flatten_to_single_artifacts,
     for_paths,
     RunnableType,
 )
 
 
 def test_runnables(ctx, runnables, original_paths=None, **kwds):
     """Return exit code indicating test or failure."""
+    if kwds.get("serve"):
+        if "galaxy" not in kwds["engine"]:
+            raise ValueError("The serve option is only supported by Galaxy-based engines.")
+        kwds["galaxy_url"] = kwds["galaxy_url"] or ''.join(("http://", kwds["host"], ":", str(kwds["port"])))
+        kwds["galaxy_admin_key"] = kwds["galaxy_admin_key"] or DEFAULT_ADMIN_API_KEY
+        pid = os.fork()
+        if pid == 0:
+            # wait for served Galaxy instance to start
+            sleep(kwds["galaxy_url"], verbose=ctx.verbose, timeout=500)
+            # then proceed to test against it
+            kwds["engine"] = "external_galaxy"
+        else:
+            # serve Galaxy instance
+            galaxy_serve(ctx, runnables, **kwds)
+            exit(1)
+
     engine_type = kwds["engine"]
     test_engine_testable = {RunnableType.galaxy_tool, RunnableType.galaxy_datamanager, RunnableType.directory}
     enable_test_engines = any(r.type not in test_engine_testable for r in runnables)
     enable_test_engines = enable_test_engines or engine_type != "galaxy"
+
     if enable_test_engines:
+        runnables = flatten_to_single_artifacts(runnables)  # the test engines cannot deal with directories
         ctx.vlog("Using test engine type %s" % engine_type)
         with engine_context(ctx, **kwds) as engine:
             test_data = engine.test(runnables)

planemo/options.py

@@ -881,6 +881,15 @@ def no_cleanup_option():
     )
 
 
+def serve_option():
+    return planemo_option(
+        "--serve",
+        is_flag=True,
+        default=False,
+        help=("Continue serving Galaxy instance after testing. Like the serve command itself, this is only compatible with Galaxy-based engines.")
+    )
+
+
 def docker_enable_option():
     return planemo_option(
         "--docker/--no_docker",

planemo/runnable.py

@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 
 import abc
+import glob
 import os
 from distutils.dir_util import copy_tree
 from enum import auto, Enum
@@ -184,7 +185,7 @@ def workfow_dir_runnables(path, return_all=False):
             return runnables[0]
 
 
-def for_path(path, temp_path=None, return_all=False):
+def for_path(path, temp_path=None, return_all=False, suppress_error=False):  # noqa C901
     """Produce a class:`Runnable` for supplied path."""
     runnable_type = None
     if os.path.isdir(path):
@@ -215,7 +216,8 @@ def for_path(path, temp_path=None, return_all=False):
             pass
 
     if runnable_type is None:
-        error("Unable to determine runnable type for path [%s]" % path)
+        if not suppress_error:
+            error("Unable to determine runnable type for path [%s]" % path)
         raise ExitCodeException(EXIT_CODE_UNKNOWN_FILE_TYPE)
 
     if temp_path:
@@ -470,6 +472,27 @@ def get_outputs(runnable, gi=None):
         raise NotImplementedError("Getting outputs for this artifact type is not yet supported.")
 
 
+def flatten_to_single_artifacts(runnables):
+    """
+    In a list of Runnables, replace non-single artifacts (currently
+    only directories) with child Runnables.
+    """
+    single_runnables = []
+    for runnable in runnables:
+        if runnable.type.is_single_artifact:
+            single_runnables.append(runnable)
+        else:
+            paths = glob.glob(f'{runnable.path}/*')
+            for path in paths:
+                if os.path.isdir(path):
+                    continue
+                try:
+                    single_runnables.append(for_path(path, suppress_error=True))
+                except ExitCodeException:  # just ignore any other files
+                    continue
+    return single_runnables
+
+
 @add_metaclass(abc.ABCMeta)
 class RunnableOutput(object):
     """Description of a single output of an execution of a Runnable."""

tests/test_cmd_serve.py

@@ -164,6 +164,33 @@ def test_shed_serve(self):
         assert found, "Failed to find fastqc id in %s" % tool_ids
         kill_pid_file(self._pid_file)
 
+    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    def test_workflow_test_with_serve(self):
+        """
+        Test testing a Galaxy workflow with the serve flag. Even though
+        this is technically using the test subcommand it is easier to test here
+        so we can use the UsesServeCommand methods
+        """
+        cat = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "cat.xml")
+        test_artifact = os.path.join(TEST_DATA_DIR, "wf2.ga")
+        extra_args = [
+            "--serve",
+            test_artifact,
+            "--port", str(self._port),
+            "--no_dependency_resolution",
+            "--extra_tools", cat,
+        ]
+        self._launch_thread_and_wait(self._run_test, extra_args)
+        time.sleep(30)
+
+        # access the served instance to check everything worked
+        user_gi = self._user_gi
+        workflows = user_gi.workflows.get_workflows()
+        assert len(workflows) == 1
+        assert workflows[0]['name'] == 'TestWorkflow1'
+        histories = user_gi.histories.get_histories(name="CWL Target History")
+        assert len(histories) == 1
+
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
     def test_serve_profile(self):
         self._test_serve_profile()
@@ -194,3 +221,6 @@ def _test_serve_profile(self, *db_options):
 
     def _run_shed(self, serve_args=[]):
         return self._run(serve_args=serve_args, serve_cmd="shed_serve")
+
+    def _run_test(self, serve_args=[]):
+        return self._run(serve_args=serve_args, serve_cmd="test")

tests/test_galaxy_config.py

@@ -49,7 +49,7 @@ def test_refgenie_config_version():
         with open(version_path, 'w') as version_fh:
             version_fh.write('VERSION_MAJOR = "21.05"')
         refgenie_config = get_refgenie_config(galaxy_root=tdc.temp_directory, refgenie_dir='/')
-    assert yaml.load(refgenie_config)['config_version'] == 0.3
+    assert yaml.load(refgenie_config, Loader=yaml.SafeLoader)['config_version'] == 0.3
 
 
 def _assert_property_is(config, prop, value):