From c548756aacc1f38068034997628db073d072105c Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Fri, 22 Apr 2022 13:50:41 +0200 Subject: [PATCH 01/10] Fix error when getting destinations by tag --- lib/galaxy/jobs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index ec4e6adc6c91..e1c7215b71f6 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -784,7 +784,7 @@ def get_destinations(self, id_or_tag): Destinations are not deepcopied, so they should not be passed to anything which might modify them. """ - return self.destinations.get(id_or_tag, None) + return self.destinations.get(id_or_tag, []) def get_job_runner_plugins(self, handler_id): """Load all configured job runner plugins From 52808823a1b674fb5236beaa83baeef1a7bb01b7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 2 May 2022 12:17:29 +0200 Subject: [PATCH 02/10] Type annotations for get_destinations --- lib/galaxy/jobs/__init__.py | 10 ++++++++-- lib/galaxy/structured_app.py | 5 +++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index e1c7215b71f6..295d0d381db0 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -13,7 +13,13 @@ import time import traceback from json import loads -from typing import Any, Dict, List, TYPE_CHECKING +from typing import ( + Any, + Dict, + Iterable, + List, + TYPE_CHECKING, +) import packaging.version import yaml @@ -773,7 +779,7 @@ def get_destination(self, id_or_tag): id_or_tag = self.default_destination_id return copy.deepcopy(self._get_single_item(self.destinations[id_or_tag])) - def get_destinations(self, id_or_tag): + def get_destinations(self, id_or_tag) -> Iterable[JobDestination]: """Given a destination ID or tag, return all JobDestinations matching the provided ID or tag :param id_or_tag: A destination ID or tag. diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index c3ff1e975319..c1bf57d029b9 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -27,6 +27,7 @@ from galaxy.workflow.trs_proxy import TrsProxy if TYPE_CHECKING: + from galaxy.jobs import JobConfiguration from galaxy.tools.data import ToolDataTableManager @@ -83,7 +84,7 @@ class MinimalManagerApp(MinimalApp): role_manager: Any # 'galaxy.managers.roles.RoleManager' installed_repository_manager: Any # 'galaxy.tool_shed.galaxy_install.installed_repository_manager.InstalledRepositoryManager' user_manager: Any - job_config: Any # 'galaxy.jobs.JobConfiguration' + job_config: "JobConfiguration" job_manager: Any # galaxy.jobs.manager.JobManager @property @@ -138,7 +139,7 @@ class StructuredApp(MinimalManagerApp): installed_repository_manager: Any # 'galaxy.tool_shed.galaxy_install.installed_repository_manager.InstalledRepositoryManager' workflow_scheduling_manager: Any # 'galaxy.workflow.scheduling_manager.WorkflowSchedulingManager' interactivetool_manager: Any - job_config: Any # 'galaxy.jobs.JobConfiguration' + job_config: "JobConfiguration" job_manager: Any # galaxy.jobs.manager.JobManager user_manager: Any api_keys_manager: Any # 'galaxy.managers.api_keys.ApiKeyManager' From 1c5c5b650c02167bc106efd3281897269617f97d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 5 May 2022 10:52:51 +0200 Subject: [PATCH 03/10] Fix download of composite files via API downloads --- lib/galaxy/datatypes/data.py | 2 +- lib/galaxy/util/zipstream.py | 5 +---- lib/galaxy/webapps/galaxy/api/datasets.py | 3 +++ lib/galaxy/webapps/galaxy/api/history_contents.py | 8 ++------ lib/galaxy/webapps/galaxy/controllers/dataset.py | 8 +++++++- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 32cd05c41b9b..8dcf40555547 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -347,7 +347,7 @@ def _archive_composite_dataset(self, trans, data, headers: Headers, do_action='z continue if not error: headers.update(archive.get_headers()) - return archive.response(), headers + return archive, headers return trans.show_error_message(msg), headers def __archive_extra_files_path(self, extra_files_path): diff --git a/lib/galaxy/util/zipstream.py b/lib/galaxy/util/zipstream.py index e467d5ab5860..c6b0cfa061e9 100644 --- a/lib/galaxy/util/zipstream.py +++ b/lib/galaxy/util/zipstream.py @@ -20,10 +20,7 @@ def response(self): if self.upstream_mod_zip: yield "\n".join(self.files).encode() else: - yield iter(self.archive) - - def get_iterator(self): - return iter(self.archive) + yield from iter(self.archive) def get_headers(self): headers = {} diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 569049ba4f32..31270033b521 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -39,6 +39,7 @@ DatasetSourceType, UpdateDatasetPermissionsPayload, ) +from galaxy.util.zipstream import ZipstreamWrapper from galaxy.webapps.galaxy.api.common import ( get_filter_query_params, get_query_parameters_from_request_excluding, @@ -256,6 +257,8 @@ def display( file_name = getattr(display_data, "name", None) if file_name: return FileResponse(file_name, headers=headers) + elif isinstance(display_data, ZipstreamWrapper): + return StreamingResponse(display_data.response(), headers=headers) return StreamingResponse(display_data, headers=headers) @router.get( diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 8010a58ee038..6b3f871cac20 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -469,9 +469,7 @@ def download_dataset_collection( while maintaining approximate collection structure. """ archive = self.service.get_dataset_collection_archive_for_download(trans, id) - if archive.upstream_mod_zip: - return StreamingResponse(archive.response(), headers=archive.get_headers()) - return StreamingResponse(archive.get_iterator(), headers=archive.get_headers(), media_type="application/zip") + return StreamingResponse(archive.response(), headers=archive.get_headers()) @router.post( '/api/histories/{history_id}/contents/{type}s', @@ -656,9 +654,7 @@ def archive( archive = self.service.archive(trans, history_id, filter_query_params, filename, dry_run) if isinstance(archive, HistoryContentsArchiveDryRunResult): return archive - if archive.upstream_mod_zip: - return StreamingResponse(archive.response(), headers=archive.get_headers()) - return StreamingResponse(archive.get_iterator(), headers=archive.get_headers(), media_type="application/zip") + return StreamingResponse(archive.response(), headers=archive.get_headers()) @router.get( '/api/histories/{history_id}/contents/{direction}/{hid}/{limit}', diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index e85295ada695..9a6fa686b89b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -29,6 +29,7 @@ ) from galaxy.util.checkers import check_binary from galaxy.util.sanitize_html import sanitize_html +from galaxy.util.zipstream import ZipstreamWrapper from galaxy.web import form_builder from galaxy.web.framework.helpers import iff from galaxy.webapps.base.controller import BaseUIController, ERROR, SUCCESS, url_for, UsesExtendedMetadataMixin @@ -188,7 +189,12 @@ def display(self, trans, dataset_id=None, preview=False, filename=None, to_ext=N # Ensure ck_size is an integer before passing through to datatypes. if ck_size: ck_size = int(ck_size) - display_data, headers = data.datatype.display_data(trans, data, preview, filename, to_ext, offset=offset, ck_size=ck_size, **kwd) + display_data, headers = data.datatype.display_data( + trans, data, preview, filename, to_ext, offset=offset, ck_size=ck_size, **kwd + ) + if isinstance(display_data, ZipstreamWrapper): + trans.response.headers.update(headers) + return display_data.response() trans.response.headers.update(headers) return display_data From 51c1c769fda760717281f71cb87d2ab8cd60dc1b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 5 May 2022 11:16:26 +0200 Subject: [PATCH 04/10] Test composite file download via API --- lib/galaxy_test/api/test_datasets.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/galaxy_test/api/test_datasets.py b/lib/galaxy_test/api/test_datasets.py index 88488736c01b..8a34b1df6a0c 100644 --- a/lib/galaxy_test/api/test_datasets.py +++ b/lib/galaxy_test/api/test_datasets.py @@ -1,8 +1,11 @@ import textwrap +import zipfile +from io import BytesIO from galaxy_test.base.populators import ( DatasetCollectionPopulator, DatasetPopulator, + skip_without_datatype, skip_without_tool, ) from ._framework import ApiTestCase @@ -200,3 +203,24 @@ def test_update_datatype(self): f"histories/{self.history_id}/contents/{hda_id}", data={'datatype': 'invalid'}, json=True) self._assert_status_code_is(invalidly_updated_hda_response, 400) + + @skip_without_datatype("velvet") + def test_composite_datatype_download(self): + item = { + "src": "composite", + "ext": "velvet", + "composite": { + "items": [ + {"src": "pasted", "paste_content": "sequences content"}, + {"src": "pasted", "paste_content": "roadmaps content"}, + {"src": "pasted", "paste_content": "log content"}, + ] + }, + } + output = self.dataset_populator.fetch_hda(self.history_id, item, wait=True) + print(output) + response = self._get(f"histories/{self.history_id}/contents/{output['id']}/display?to_ext=zip") + self._assert_status_code_is(response, 200) + archive = zipfile.ZipFile(BytesIO(response.content)) + namelist = archive.namelist() + assert len(namelist) == 4, f"Expected 3 elements in [{namelist}]" From 114bbc862d4d2fae7154a856d11dfe0c342e598c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 5 May 2022 20:00:59 +0200 Subject: [PATCH 05/10] Also update legacy route --- lib/galaxy/webapps/galaxy/api/datasets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 31270033b521..c78a12a0e0e6 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -455,6 +455,8 @@ def display(self, trans, history_content_id, history_id, trans, history_content_id, history_id, preview, filename, to_ext, raw, **kwd ) trans.response.headers.update(headers) + if isinstance(display_data, ZipstreamWrapper): + return display_data.response() return display_data @web.expose_api From 6de475c755d8c404be68d55054f53231a0ea1550 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 6 May 2022 17:33:24 +0200 Subject: [PATCH 06/10] Update gravity, fixes celery-beat in privsep setup --- lib/galaxy/config/sample/galaxy.yml.sample | 9 ++++++++- lib/galaxy/dependencies/dev-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 7bff2d9b9b39..6c46e297be12 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -25,7 +25,7 @@ gravity: # galaxy_root: # Set to a directory that should contain log files for the processes controlled by Gravity. - # If not specified defaults to ``/logs``. + # If not specified defaults to ``/log``. # log_dir: # Set to Galaxy's virtualenv directory. @@ -77,6 +77,13 @@ gravity: # Valid options are: DEBUG, INFO, WARNING, ERROR # loglevel: DEBUG + # Queues to join + # queues: celery,galaxy.internal,galaxy.external + + # Pool implementation + # Valid options are: prefork, eventlet, gevent, solo, processes, threads + # pool: threads + # Extra arguments to pass to Celery command line. # extra_args: diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 36425698bf47..72b5de8b3383 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -68,7 +68,7 @@ fs==2.4.14 funcsigs==1.0.2 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") galaxy-sequence-utils==1.1.5 -gravity==0.11.0; python_version >= "3.6" +gravity==0.12.0; python_version >= "3.6" greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0" gunicorn==20.1.0; python_version >= "3.5" gxformat2==0.15.0 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 04817885bd19..1f711818b4b6 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -56,7 +56,7 @@ fs==2.4.14 funcsigs==1.0.2 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") galaxy-sequence-utils==1.1.5 -gravity==0.11.0; python_version >= "3.6" +gravity==0.12.0; python_version >= "3.6" greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0" gunicorn==20.1.0; python_version >= "3.5" gxformat2==0.15.0 From 8f86ba8fff1db4c839e0a8dcc049cb2ebceb40c9 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 7 May 2022 17:54:13 +0200 Subject: [PATCH 07/10] replace decodestring with decodeybtes --- lib/galaxy/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7fb62d5f95fd..1999eb03b557 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7712,7 +7712,7 @@ def store(cls, server_url, association): assoc = cls.sa_session.query(cls).filter_by(server_url=server_url, handle=association.handle)[0] except IndexError: assoc = cls(server_url=server_url, handle=association.handle) - assoc.secret = base64.encodestring(association.secret).decode() + assoc.secret = base64.encodebytes(association.secret).decode() assoc.issued = association.issued assoc.lifetime = association.lifetime assoc.assoc_type = association.assoc_type From 9dec207a5251086a09236ea5f1751f9519a0a41d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 6 Apr 2022 23:47:54 +0100 Subject: [PATCH 08/10] Respect GRAVITY_STATE_DIR when activating venv So that: export GRAVITY_STATE_DIR=/srv/galaxy/gravity ./run.sh writes gravity state/log files to the right place. --- scripts/common_startup_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index fb296401c53d..d5079e4c99de 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -137,7 +137,7 @@ setup_gravity_state_dir() { echo "Setting \$GRAVITY_STATE_DIR in ${GALAXY_VIRTUAL_ENV}/bin/activate" echo '' >> "${GALAXY_VIRTUAL_ENV}/bin/activate" echo '# Galaxy Gravity per-instance state directory configured by Galaxy common_startup.sh' >> "${GALAXY_VIRTUAL_ENV}/bin/activate" - echo "GRAVITY_STATE_DIR='$(pwd)/database/gravity'" >> "${GALAXY_VIRTUAL_ENV}/bin/activate" + echo "GRAVITY_STATE_DIR=\${GRAVITY_STATE_DIR:-'$(pwd)/database/gravity'}" >> "${GALAXY_VIRTUAL_ENV}/bin/activate" echo 'export GRAVITY_STATE_DIR' >> "${GALAXY_VIRTUAL_ENV}/bin/activate" fi } From 853ef69d0240010a7a9816bccd9185313a6b771e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 11 May 2022 10:25:51 +0200 Subject: [PATCH 09/10] Redirect stderr of trap cleanup Prevents ``` rm: cannot remove '/data/jwd/main/046/709/46709070/tmp/out.471762': No such file or directory rm: cannot remove '/data/jwd/main/046/709/46709070/tmp/err.471762': No such file or directory ``` reported in https://github.com/galaxyproject/galaxy/issues/13902. I don't think this is what fails the job, there is likely something else wrong on top of this. --- lib/galaxy/jobs/command_factory.py | 2 +- test/unit/app/jobs/test_command_factory.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 896c1caf1617..0e9b2eac6092 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -282,7 +282,7 @@ def capture_stdout_stderr(self, stdout_file, stderr_file): if TRAP_KILL_CONTAINER in self.commands: # We need to replace the container kill trap with one that removes the named pipes and kills the container self.commands = self.commands.replace(TRAP_KILL_CONTAINER, "") - trap_command = """trap 'rm "$__out" "$__err"; _on_exit' EXIT""" + trap_command = """trap 'rm "$__out" "$__err" 2> /dev/null || true; _on_exit' EXIT""" self.prepend_command( f"""__out="${{TMPDIR:-.}}/out.$$" __err="${{TMPDIR:-.}}/err.$$" mkfifo "$__out" "$__err" diff --git a/test/unit/app/jobs/test_command_factory.py b/test/unit/app/jobs/test_command_factory.py index 4fa4d45b85f0..7470454a4924 100644 --- a/test/unit/app/jobs/test_command_factory.py +++ b/test/unit/app/jobs/test_command_factory.py @@ -54,7 +54,7 @@ def test_kill_trap_replaced(self): self.job_wrapper.command_line = f"{TRAP_KILL_CONTAINER}{MOCK_COMMAND_LINE}" expected_command_line = self._surround_command(MOCK_COMMAND_LINE).replace( """trap 'rm "$__out" "$__err"' EXIT""", - """trap 'rm "$__out" "$__err"; _on_exit' EXIT""" + """trap 'rm "$__out" "$__err" 2> /dev/null || true; _on_exit' EXIT""" ) self.__assert_command_is(expected_command_line) From 8f903c9e44f521f87520fdec6dfe875c6196b5a4 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 11 May 2022 13:11:25 +0200 Subject: [PATCH 10/10] Set default celery app for all threads --- lib/galaxy/celery/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 5bc16fb0d966..c987453960a8 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -72,6 +72,7 @@ def get_history_audit_table_prune_interval(): broker = get_broker() celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks']) +celery_app.set_default() prune_interval = get_history_audit_table_prune_interval() if prune_interval > 0: celery_app.conf.beat_schedule = {