From 4667cec17b9cf0786696378bb10535d05c35fbaa Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 30 Aug 2024 16:30:49 +0200 Subject: [PATCH 1/6] Add regression test for subworkflow that stops scheduling --- lib/galaxy_test/api/test_workflows.py | 24 ++++++++++++++++++++++++ lib/galaxy_test/base/populators.py | 11 +++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 519281ded0a9..8c9b7187006d 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -2455,6 +2455,30 @@ def test_run_nested_conditional_workflow_steps(self): if step["workflow_step_label"] == "cat1": assert sum(1 for j in step["jobs"] if j["state"] == "skipped") == 1 + def test_run_workflow_conditional_subworkflow_step_with_hdca_creation(self): + # Regression test, ensures scheduling proceeds even if a skipped step creates a collection + with self.dataset_populator.test_history() as history_id: + self._run_workflow( + """ +class: GalaxyWorkflow +inputs: [] +steps: + conditional_subworkflow_step: + when: $(false) + run: + class: GalaxyWorkflow + inputs: [] + steps: + create_collection: + tool_id: create_input_collection + flatten_collection: + tool_id: cat_list + in: + input1: create_collection/output + """, + history_id=history_id, + ) + def test_run_workflow_conditional_step_map_over_expression_tool(self): with self.dataset_populator.test_history() as history_id: summary = self._run_workflow( diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index d373b23d389d..0347fbdfadb2 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -1770,7 +1770,11 @@ def upload_yaml_workflow(self, yaml_content: YamlContentT, **kwds) -> str: return workflow_id def wait_for_invocation( - self, workflow_id: str, invocation_id: str, timeout: timeout_type = DEFAULT_TIMEOUT, assert_ok: bool = True + self, + workflow_id: Optional[str], + invocation_id: str, + timeout: timeout_type = DEFAULT_TIMEOUT, + assert_ok: bool = True, ) -> str: url = f"invocations/{invocation_id}" @@ -1818,7 +1822,7 @@ def invocation_count(): def wait_for_workflow( self, - workflow_id: str, + workflow_id: Optional[str], invocation_id: str, history_id: str, assert_ok: bool = True, @@ -1827,6 +1831,9 @@ def wait_for_workflow( """Wait for a workflow invocation to completely schedule and then history to be complete.""" self.wait_for_invocation(workflow_id, invocation_id, timeout=timeout, assert_ok=assert_ok) + for step in self.get_invocation(invocation_id)["steps"]: + if step["subworkflow_invocation_id"]: + self.wait_for_invocation(None, step["subworkflow_invocation_id"], timeout=timeout, assert_ok=assert_ok) self.dataset_populator.wait_for_history_jobs(history_id, assert_ok=assert_ok, timeout=timeout) def get_invocation(self, invocation_id, step_details=False): From 96e7487bef959a74d043dec1de94c2d0c6cbbcbf Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 30 Aug 2024 16:03:48 +0200 Subject: [PATCH 2/6] Also mark direct HDCA outputs as skipped --- lib/galaxy/tools/actions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 1a16ed58af3c..826b5d54059a 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -681,6 +681,7 @@ def handle_output(name, output, hidden=None): output_collection.mark_as_populated() for hdca in output_collections.out_collection_instances.values(): hdca.visible = False + hdca.collection.mark_as_populated() object_store_populator = ObjectStorePopulator(trans.app, trans.user) for data in out_data.values(): data.set_skipped(object_store_populator) From 3db1055ad0e2cab709ffc573efceb19e4db90950 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 30 Aug 2024 17:51:00 +0200 Subject: [PATCH 3/6] Fix directory get or create logic Fixes https://sentry.galaxyproject.org/share/issue/d7131c55397947f3b81d64afa4674895/: ``` FileExistsError: [Errno 17] File exists: '/srv/galaxy/main/var/tool_search_index/all_tools' File "main.py", line 112, in app_loop galaxy_app = load_galaxy_app( File "main.py", line 91, in load_galaxy_app app = UniverseApplication(global_conf=config_builder.global_conf(), attach_to_pools=attach_to_pools, **kwds) File "galaxy/app.py", line 773, in __init__ self._configure_toolbox() File "galaxy/app.py", line 391, in _configure_toolbox ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help), File "galaxy/tools/search/__init__.py", line 99, in __init__ panel_searches[panel_view_id] = ToolPanelViewSearch( File "galaxy/tools/search/__init__.py", line 200, in __init__ self.index = self._index_setup() File "galaxy/tools/search/__init__.py", line 204, in _index_setup return get_or_create_index(self.index_dir, self.schema) File "galaxy/tools/search/__init__.py", line 75, in get_or_create_index os.makedirs(index_dir) File "", line 225, in makedirs ``` --- lib/galaxy/tools/search/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 3fcce2caab42..9fa0c24baff4 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -71,8 +71,7 @@ def get_or_create_index(index_dir, schema): """Get or create a reference to the index.""" - if not os.path.exists(index_dir): - os.makedirs(index_dir) + os.makedirs(index_dir, exist_ok=True) if index.exists_in(index_dir): idx = index.open_dir(index_dir) if idx.schema == schema: From 1ec3297b5e177a6d0c788e2173c4b177f775b587 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 30 Aug 2024 18:25:17 +0200 Subject: [PATCH 4/6] Fix intermediateModalTitle --- .../components/Workflow/Run/WorkflowStorageConfiguration.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Workflow/Run/WorkflowStorageConfiguration.vue b/client/src/components/Workflow/Run/WorkflowStorageConfiguration.vue index d67dc2b5a191..893ecc310dcd 100644 --- a/client/src/components/Workflow/Run/WorkflowStorageConfiguration.vue +++ b/client/src/components/Workflow/Run/WorkflowStorageConfiguration.vue @@ -91,7 +91,7 @@ export default { return `Invocation ${this.preferredOrEmptyString} Storage Location`; }, intermediateModalTitle() { - return `Invocation {{ preferredOrEmptyString }} Storage Location (Intermediate Datasets)`; + return `Invocation ${this.preferredOrEmptyString} Storage Location (Intermediate Datasets)`; }, suffixPrimary() { if (this.splitObjectStore) { From f1dddfb4b1377566746764ce33a8a6d31b524739 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 30 Aug 2024 18:43:54 +0200 Subject: [PATCH 5/6] Fix job summary for optional unset job data inputs Fixes https://sentry.galaxyproject.org/share/issue/31c83cea41a94a7c9c367a66d89fae37/: ``` AttributeError: 'NoneType' object has no attribute 'id' File "starlette/applications.py", line 123, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/middleware/errors.py", line 186, in __call__ raise exc File "starlette/middleware/errors.py", line 164, in __call__ await self.app(scope, receive, _send) File "starlette_context/middleware/raw_middleware.py", line 92, in __call__ await self.app(scope, receive, send_wrapper) File "starlette/middleware/base.py", line 189, in __call__ with collapse_excgroups(): File "contextlib.py", line 155, in __exit__ self.gen.throw(typ, value, traceback) File "starlette/_utils.py", line 93, in collapse_excgroups raise exc File "starlette/middleware/base.py", line 191, in __call__ response = await self.dispatch_func(request, call_next) File "galaxy/webapps/galaxy/fast_app.py", line 109, in add_x_frame_options response = await call_next(request) File "starlette/middleware/base.py", line 165, in call_next raise app_exc File "starlette/middleware/base.py", line 151, in coro await self.app(scope, receive_or_disconnect, send_no_error) File "starlette/middleware/exceptions.py", line 65, in __call__ await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 756, in __call__ await self.middleware_stack(scope, receive, send) File "starlette/routing.py", line 776, in app await route.handle(scope, receive, send) File "starlette/routing.py", line 297, in handle await self.app(scope, receive, send) File "starlette/routing.py", line 77, in app await wrap_app_handling_exceptions(app, request)(scope, receive, send) File "starlette/_exception_handler.py", line 64, in wrapped_app raise exc File "starlette/_exception_handler.py", line 53, in wrapped_app await app(scope, receive, sender) File "starlette/routing.py", line 72, in app response = await func(request) File "fastapi/routing.py", line 278, in app raw_response = await run_endpoint_function( File "fastapi/routing.py", line 193, in run_endpoint_function return await run_in_threadpool(dependant.call, **values) File "starlette/concurrency.py", line 42, in run_in_threadpool return await anyio.to_thread.run_sync(func, *args) File "anyio/to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( File "anyio/_backends/_asyncio.py", line 2144, in run_sync_in_worker_thread return await future File "anyio/_backends/_asyncio.py", line 851, in run result = context.run(func, *args) File "galaxy/webapps/galaxy/api/jobs.py", line 410, in parameters_display_by_dataset return summarize_job_parameters(trans, job) File "galaxy/managers/jobs.py", line 1055, in summarize_job_parameters parameters = inputs_recursive(tool.inputs, params_objects, depth=1, upgrade_messages=upgrade_messages) File "galaxy/managers/jobs.py", line 994, in inputs_recursive element_id = element.id ``` which you can reproduce by connecting an optional dataset and a required dataset to a multiple="true" data parameter and then looking at the tool input parameters. --- client/src/api/schema/schema.ts | 2 +- .../JobParameters/JobParametersArrayValue.vue | 3 ++- lib/galaxy/managers/jobs.py | 13 ++++++++----- lib/galaxy/schema/jobs.py | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index f878a162098a..9abd677c7b27 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -9132,7 +9132,7 @@ export interface components { * @description The values of the job parameter */ value?: - | components["schemas"]["EncodedJobParameterHistoryItem"][] + | (components["schemas"]["EncodedJobParameterHistoryItem"] | null)[] | number | number | boolean diff --git a/client/src/components/JobParameters/JobParametersArrayValue.vue b/client/src/components/JobParameters/JobParametersArrayValue.vue index 31be3694434a..014d0e6d12de 100644 --- a/client/src/components/JobParameters/JobParametersArrayValue.vue +++ b/client/src/components/JobParameters/JobParametersArrayValue.vue @@ -1,8 +1,9 @@