From 3a2bc2cf251e649703b1eda843b77aa99bd2bf5f Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 6 Aug 2024 14:36:57 -0400 Subject: [PATCH 01/10] Strip whitespace when listifying admin users --- lib/galaxy/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index ca78874969eb..9b2d435ae393 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -594,7 +594,7 @@ def admin_users(self): @admin_users.setter def admin_users(self, value): self._admin_users = value - self.admin_users_list = listify(value) + self.admin_users_list = listify(value, do_strip=True) def is_admin_user(self, user: Optional["User"]) -> bool: """Determine if the provided user is listed in `admin_users`.""" From f30b21318618edabe313e62cecc861feaaaa7bb6 Mon Sep 17 00:00:00 2001 From: Alireza Heidari Date: Wed, 18 Sep 2024 12:32:30 +0200 Subject: [PATCH 02/10] =?UTF-8?q?=E2=9C=A8:=20=20added=20`is:bookmarked`?= =?UTF-8?q?=20filter=20to=20workflows=20index=20API=20`"/api/workflows"`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/src/api/schema/schema.ts | 9 +++++++++ lib/galaxy/managers/workflows.py | 6 ++++++ lib/galaxy/webapps/galaxy/api/workflows.py | 12 ++++++++++++ 3 files changed, 27 insertions(+) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index af2fc0a5b8bb..eb60f571e10c 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -24368,9 +24368,18 @@ export interface operations { * `is:published` * : Include only published workflows in the final result. Be sure the query parameter `show_published` is set to `true` if to include all published workflows and not just the requesting user's. * + * `is:importable` + * : Include only importable workflows in the final result. + * + * `is:deleted` + * : Include only deleted workflows in the final result. + * * `is:share_with_me` * : Include only workflows shared with the requesting user. Be sure the query parameter `show_shared` is set to `true` if to include shared workflows. * + * `is:bookmarked` + * : Include only workflows bookmarked by the requesting user. + * * ## Free Text * * Free text search terms will be searched against the following attributes of the diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 0d970de24293..3510c9cbbf0b 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -239,6 +239,12 @@ def name_filter(term): message = "Can only use tag is:shared_with_me if show_shared parameter also true." raise exceptions.RequestParameterInvalidException(message) stmt = stmt.where(StoredWorkflowUserShareAssociation.user == user) + elif q == "bookmarked": + stmt = ( + stmt.outerjoin(model.StoredWorkflowMenuEntry) + .where(model.StoredWorkflowMenuEntry.stored_workflow_id == StoredWorkflow.id) + .where(model.StoredWorkflowMenuEntry.user_id == user.id) + ) elif isinstance(term, RawTextTerm): tf = w_tag_filter(term.text, False) alias = aliased(User) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 64d1d2a4c0ba..bb3b7f2ae880 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -854,10 +854,22 @@ def __get_stored_workflow(self, trans, workflow_id, **kwd): "is:published", "Include only published workflows in the final result. Be sure the query parameter `show_published` is set to `true` if to include all published workflows and not just the requesting user's.", ), + IndexQueryTag( + "is:importable", + "Include only importable workflows in the final result.", + ), + IndexQueryTag( + "is:deleted", + "Include only deleted workflows in the final result.", + ), IndexQueryTag( "is:share_with_me", "Include only workflows shared with the requesting user. Be sure the query parameter `show_shared` is set to `true` if to include shared workflows.", ), + IndexQueryTag( + "is:bookmarked", + "Include only workflows bookmarked by the requesting user.", + ), ] SearchQueryParam: Optional[str] = search_query_param( From c1abcd7805e4e4968f188ce2549e44580557bf96 Mon Sep 17 00:00:00 2001 From: Alireza Heidari Date: Wed, 18 Sep 2024 12:33:59 +0200 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=90=9B:=20updated=20bookmarked=20fi?= =?UTF-8?q?lter=20in=20`WorkflowList`=20regarding=20to=20the=20new=20searc?= =?UTF-8?q?h=20query?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/Workflow/WorkflowFilters.js | 7 ++++++ .../src/components/Workflow/WorkflowList.vue | 22 +++++-------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/client/src/components/Workflow/WorkflowFilters.js b/client/src/components/Workflow/WorkflowFilters.js index 1280b6e450a5..a7c1f3f3b3c8 100644 --- a/client/src/components/Workflow/WorkflowFilters.js +++ b/client/src/components/Workflow/WorkflowFilters.js @@ -116,6 +116,13 @@ export function WorkflowFilters(activeList = "my") { handler: equals("deleted", "deleted", toBool), menuItem: true, }, + bookmarked: { + placeholder: "Bookmarked", + type: Boolean, + boolType: "is", + handler: equals("bookmarked", "bookmarked", toBool), + menuItem: true, + }, }, undefined, false, diff --git a/client/src/components/Workflow/WorkflowList.vue b/client/src/components/Workflow/WorkflowList.vue index 3cbda017cc01..5de798bf5807 100644 --- a/client/src/components/Workflow/WorkflowList.vue +++ b/client/src/components/Workflow/WorkflowList.vue @@ -44,7 +44,6 @@ const overlay = ref(false); const filterText = ref(""); const totalWorkflows = ref(0); const showAdvanced = ref(false); -const showBookmarked = ref(false); const listHeader = ref(null); const workflowsLoaded = ref([]); @@ -65,6 +64,7 @@ const searchPlaceHolder = computed(() => { const published = computed(() => props.activeList === "published"); const sharedWithMe = computed(() => props.activeList === "shared_with_me"); const showDeleted = computed(() => filterText.value.includes("is:deleted")); +const showBookmarked = computed(() => filterText.value.includes("is:bookmarked")); const currentPage = computed(() => Math.floor(offset.value / limit.value) + 1); const view = computed(() => (userStore.preferredListViewMode as ListView) || "grid"); const sortDesc = computed(() => (listHeader.value && listHeader.value.sortDesc) ?? true); @@ -91,17 +91,12 @@ function updateFilterValue(filterKey: string, newValue: any) { filterText.value = workflowFilters.value.setFilterValue(currentFilterText, filterKey, newValue); } -function toggleBookmarked(bookmarked?: boolean) { - showBookmarked.value = bookmarked ?? !showBookmarked.value; -} - function onToggleBookmarked() { - toggleBookmarked(); + updateFilterValue("bookmarked", true); } function onToggleDeleted() { updateFilterValue("deleted", true); - toggleBookmarked(false); } async function load(overlayLoading = false, silent = false) { @@ -141,9 +136,7 @@ async function load(overlayLoading = false, silent = false) { skipStepCounts: true, }); - let filteredWorkflows = showBookmarked.value - ? filter(data, (workflow: any) => workflow.show_in_tool_panel) - : data; + let filteredWorkflows = data; if (props.activeList === "my") { filteredWorkflows = filter(filteredWorkflows, (w: any) => w.owner === userStore.currentUser?.username); @@ -151,11 +144,7 @@ async function load(overlayLoading = false, silent = false) { workflowsLoaded.value = filteredWorkflows; - if (showBookmarked.value) { - totalWorkflows.value = filteredWorkflows.length; - } else { - totalWorkflows.value = parseInt(headers.get("Total_matches") || "0", 10) || 0; - } + totalWorkflows.value = parseInt(headers.get("Total_matches") || "0", 10) || 0; } catch (e) { Toast.error(`Failed to load workflows: ${e}`); } finally { @@ -181,7 +170,7 @@ function validatedFilterText() { return workflowFilters.value.getFilterText(validFilters.value, true); } -watch([filterText, sortBy, sortDesc, showBookmarked], async () => { +watch([filterText, sortBy, sortDesc], async () => { offset.value = 0; await load(true); }); @@ -261,7 +250,6 @@ onMounted(() => { size="sm" :title="bookmarkButtonTitle" :pressed="showBookmarked" - :disabled="showDeleted" variant="outline-primary" @click="onToggleBookmarked"> From c1cf7c11faaf983fb01d4ff9a27e6f51c1cd0cea Mon Sep 17 00:00:00 2001 From: Alireza Heidari Date: Wed, 18 Sep 2024 16:16:30 +0200 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F:=20use=20`join`=20i?= =?UTF-8?q?nstead=20of=20`outerjoin`=20in=20workflow=20list=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marius van den Beek --- lib/galaxy/managers/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 3510c9cbbf0b..eabaf60af260 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -241,7 +241,7 @@ def name_filter(term): stmt = stmt.where(StoredWorkflowUserShareAssociation.user == user) elif q == "bookmarked": stmt = ( - stmt.outerjoin(model.StoredWorkflowMenuEntry) + stmt.join(model.StoredWorkflowMenuEntry) .where(model.StoredWorkflowMenuEntry.stored_workflow_id == StoredWorkflow.id) .where(model.StoredWorkflowMenuEntry.user_id == user.id) ) From 5ead32b03f8bee1e32584da031e1802dbc4ee161 Mon Sep 17 00:00:00 2001 From: Alireza Heidari Date: Thu, 19 Sep 2024 12:10:30 +0200 Subject: [PATCH 05/10] =?UTF-8?q?=F0=9F=8E=A8:=20fix=20workflow=20list=20i?= =?UTF-8?q?ndex=20query=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ahmed Hamid Awan --- lib/galaxy/webapps/galaxy/api/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index bb3b7f2ae880..b0dbf0d8c504 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -863,7 +863,7 @@ def __get_stored_workflow(self, trans, workflow_id, **kwd): "Include only deleted workflows in the final result.", ), IndexQueryTag( - "is:share_with_me", + "is:shared_with_me", "Include only workflows shared with the requesting user. Be sure the query parameter `show_shared` is set to `true` if to include shared workflows.", ), IndexQueryTag( From b47a8b0da46c4e40c175ae5e6cae29f6b40dc26c Mon Sep 17 00:00:00 2001 From: Alireza Heidari Date: Thu, 19 Sep 2024 15:12:38 +0200 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F:=20update=20client?= =?UTF-8?q?=20api=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/src/api/schema/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index eb60f571e10c..0659ad0d08d9 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -24374,7 +24374,7 @@ export interface operations { * `is:deleted` * : Include only deleted workflows in the final result. * - * `is:share_with_me` + * `is:shared_with_me` * : Include only workflows shared with the requesting user. Be sure the query parameter `show_shared` is set to `true` if to include shared workflows. * * `is:bookmarked` From e1d6c93a21ae0f89932e82a0be0255192be39421 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Sep 2024 16:53:39 +0200 Subject: [PATCH 07/10] Fix discovered outputs with directory metadata and distributed object stores The problem is that `object_store.is_private` would check in which object store the dataset is stored, and that fails if we haven't written to the object store yet (which is why extended metadata is a workaround). Fixes https://github.com/galaxyproject/galaxy/issues/17208 --- lib/galaxy/model/store/discover.py | 18 ++++++++++----- test/integration/objectstore/test_jobs.py | 26 ++++++++++++++++++---- test/integration/test_extended_metadata.py | 1 + 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 17389c7b83eb..db3ac40bf2e5 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -132,11 +132,6 @@ def create_dataset( ) self.persist_object(primary_data) - if init_from: - self.permission_provider.copy_dataset_permissions(init_from, primary_data) - primary_data.state = init_from.state - else: - self.permission_provider.set_default_hda_permissions(primary_data) else: ld = galaxy.model.LibraryDataset(folder=library_folder, name=name) ldda = galaxy.model.LibraryDatasetDatasetAssociation( @@ -208,6 +203,7 @@ def create_dataset( filename=filename, link_data=link_data, output_name=output_name, + init_from=init_from, ) else: storage_callbacks.append( @@ -218,11 +214,14 @@ def create_dataset( filename=filename, link_data=link_data, output_name=output_name, + init_from=init_from, ) ) return primary_data - def finalize_storage(self, primary_data, dataset_attributes, extra_files, filename, link_data, output_name): + def finalize_storage( + self, primary_data, dataset_attributes, extra_files, filename, link_data, output_name, init_from + ): if primary_data.dataset.purged: # metadata won't be set, maybe we should do that, then purge ? primary_data.dataset.file_size = 0 @@ -243,6 +242,13 @@ def finalize_storage(self, primary_data, dataset_attributes, extra_files, filena else: # We are sure there are no extra files, so optimize things that follow by settting total size also. primary_data.set_size(no_extra_files=True) + + if init_from: + self.permission_provider.copy_dataset_permissions(init_from, primary_data) + primary_data.state = init_from.state + else: + self.permission_provider.set_default_hda_permissions(primary_data) + # TODO: this might run set_meta after copying the file to the object store, which could be inefficient if job working directory is closer to the node. self.set_datasets_metadata(datasets=[primary_data], datasets_attributes=[dataset_attributes]) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index 30383b8e5372..3d1bab41e37e 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -3,12 +3,17 @@ import os import string +from galaxy_test.driver.integration_util import ( + integration_module_instance, + integration_tool_runner, +) from ._base import ( BaseObjectStoreIntegrationTestCase, files_count, ) +from .test_selection_with_resource_parameters import DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE -DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template( +HIERARCHICAL_OBJECT_STORE_CONFIG_TEMPLATE = string.Template( """ @@ -39,7 +44,20 @@ TEST_INPUT_FILES_CONTENT = "1 2 3" -class TestObjectStoreJobsIntegration(BaseObjectStoreIntegrationTestCase): +class TestDistributedObjectStore(BaseObjectStoreIntegrationTestCase): + @classmethod + def handle_galaxy_config_kwds(cls, config): + super().handle_galaxy_config_kwds(config) + config["metadata_strategy"] = "directory" + config["object_store_store_by"] = "uuid" + cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config) + + +instance = integration_module_instance(TestDistributedObjectStore) +test_tools = integration_tool_runner(["all_output_types"]) + + +class TestObjectStoreJobsIntegration(TestDistributedObjectStore): # setup by _configure_object_store files1_path: str files2_path: str @@ -48,7 +66,7 @@ class TestObjectStoreJobsIntegration(BaseObjectStoreIntegrationTestCase): @classmethod def handle_galaxy_config_kwds(cls, config): super().handle_galaxy_config_kwds(config) - cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config) + cls._configure_object_store(HIERARCHICAL_OBJECT_STORE_CONFIG_TEMPLATE, config) def setUp(self): super().setUp() @@ -68,7 +86,7 @@ def setUp(self): def test_files_count_and_content_in_each_objectstore_backend(self): """ According to the ObjectStore configuration given in the - `DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE` variable, datasets + `HIERARCHICAL_OBJECT_STORE_CONFIG_TEMPLATE` variable, datasets can be stored on three backends, named: - primary/files1; - primary/files2; diff --git a/test/integration/test_extended_metadata.py b/test/integration/test_extended_metadata.py index d267ed01c418..8ef2c8306417 100644 --- a/test/integration/test_extended_metadata.py +++ b/test/integration/test_extended_metadata.py @@ -43,6 +43,7 @@ "collection_creates_dynamic_nested_from_json_elements", "implicit_conversion", "environment_variables", + "all_output_types", ] From 1e94e04e2bbbf39a3c329749ad4e111fcc03f1e7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Sep 2024 21:48:41 +0200 Subject: [PATCH 08/10] Fix check dataset check Followup to https://github.com/galaxyproject/galaxy/pull/18846 --- lib/galaxy/webapps/galaxy/controllers/dataset.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index f9ff2a413893..d003aedaf541 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -19,7 +19,10 @@ encode_dataset_user, ) from galaxy.datatypes.sniff import guess_ext -from galaxy.exceptions import RequestParameterInvalidException +from galaxy.exceptions import ( + InsufficientPermissionsException, + RequestParameterInvalidException, +) from galaxy.managers.hdas import ( HDADeserializer, HDAManager, @@ -106,7 +109,7 @@ def _check_dataset(self, trans, hda_id): if not data: raise web.httpexceptions.HTTPNotFound(f"Invalid reference dataset id: {str(hda_id)}.") if not self._can_access_dataset(trans, data): - return trans.show_error_message("You are not allowed to access this dataset") + raise InsufficientPermissionsException("You are not allowed to access this dataset") self.app.hda_manager.ensure_dataset_on_disk(trans, data) return data From 99b3d29e69c905996c6c26296a89650b7ee1755b Mon Sep 17 00:00:00 2001 From: Dannon Date: Fri, 20 Sep 2024 07:13:11 +0200 Subject: [PATCH 09/10] Change mechanism for loading data into editor viz avoiding injecting it as innerHTML and executing contents. --- config/plugins/visualizations/editor/templates/editor.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugins/visualizations/editor/templates/editor.mako b/config/plugins/visualizations/editor/templates/editor.mako index 385edbba6205..1046842c3417 100755 --- a/config/plugins/visualizations/editor/templates/editor.mako +++ b/config/plugins/visualizations/editor/templates/editor.mako @@ -62,11 +62,11 @@ const ajax_url = "${h.url_for( controller='/datasets', action='index')}/" + hda_id + "/display"; const data = httpGet(ajax_url); - document.getElementById("editor").innerHTML = data; var editor = ace.edit("editor", { mode: "ace/mode/powershell", theme: "ace/theme/textmate" }); + editor.setValue(data, -1); From 9a9c4a8eb85211d627d5dbe3573656eb9c22e2e3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 29 Jul 2024 10:28:44 -0400 Subject: [PATCH 10/10] Do not set uuid on dataset imports. --- lib/galaxy/model/store/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index c2a616c50419..063bbc295dd1 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -289,10 +289,11 @@ def remap_objects(p, k, obj): self._attach_raw_id_if_editing(dataset_instance, dataset_attrs) # Older style... - if 'uuid' in dataset_attrs: - dataset_instance.dataset.uuid = dataset_attrs["uuid"] - if 'dataset_uuid' in dataset_attrs: - dataset_instance.dataset.uuid = dataset_attrs["dataset_uuid"] + if self.import_options.allow_edit: + if 'uuid' in dataset_attrs: + dataset_instance.dataset.uuid = dataset_attrs["uuid"] + if 'dataset_uuid' in dataset_attrs: + dataset_instance.dataset.uuid = dataset_attrs["dataset_uuid"] self._session_add(dataset_instance) self._flush()