From 2f69f5788c4259a0d5784d44dc0a91bd26a932ba Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 17 Feb 2024 20:00:20 -0500 Subject: [PATCH] Object Store creation templates. --- client/src/api/objectStores.ts | 14 +- client/src/components/Form/FormElement.vue | 59 +-- client/src/components/Form/form-elements.scss | 58 +++ .../ObjectStore/DescribeObjectStore.vue | 6 +- .../ObjectStore/Instances/CreateForm.test.ts | 124 +++++ .../ObjectStore/Instances/CreateForm.vue | 81 ++++ .../ObjectStore/Instances/CreateInstance.vue | 34 ++ .../ObjectStore/Instances/EditInstance.vue | 90 ++++ .../ObjectStore/Instances/EditSecrets.vue | 39 ++ .../Instances/InstanceDropdown.vue | 57 +++ .../ObjectStore/Instances/InstanceForm.vue | 43 ++ .../ObjectStore/Instances/ManageIndex.vue | 106 ++++ .../ObjectStore/Instances/UpgradeForm.test.ts | 145 ++++++ .../ObjectStore/Instances/UpgradeForm.vue | 86 ++++ .../ObjectStore/Instances/UpgradeInstance.vue | 27 ++ .../ObjectStore/Instances/VaultSecret.vue | 65 +++ .../components/ObjectStore/Instances/icons.ts | 5 + .../ObjectStore/Instances/instance.ts | 25 + .../ObjectStore/Instances/routing.ts | 16 + .../ObjectStore/Instances/services.ts | 4 + .../components/ObjectStore/Instances/types.ts | 9 + .../components/ObjectStore/Instances/util.ts | 45 ++ .../ObjectStore/ObjectStoreBadge.vue | 6 +- .../ObjectStoreRestrictionSpan.vue | 8 +- .../ObjectStore/ObjectStoreTypeSpan.vue | 27 ++ .../ShowSelectedObjectStore.test.js | 28 ++ .../Templates/CreateUserObjectStore.vue | 39 ++ .../ObjectStore/Templates/SelectTemplate.vue | 50 ++ .../ObjectStore/Templates/TemplateSummary.vue | 28 ++ .../Templates/TemplateSummaryPopover.vue | 21 + .../Templates/TemplateSummarySpan.vue | 32 ++ .../components/ObjectStore/Templates/types.ts | 4 + client/src/components/ObjectStore/style.css | 4 + .../src/components/User/UserPreferences.vue | 30 +- client/src/entry/analysis/router.js | 31 ++ .../stores/objectStoreInstancesStore.test.ts | 57 +++ .../src/stores/objectStoreInstancesStore.ts | 55 +++ .../stores/objectStoreTemplatesStore.test.ts | 104 ++++ .../src/stores/objectStoreTemplatesStore.ts | 115 +++++ client/src/stores/testUtils.ts | 5 + client/src/utils/navigation/navigation.yml | 7 + client/tests/jest/helpers.js | 10 + lib/galaxy/app.py | 30 +- lib/galaxy/config/schemas/config_schema.yml | 15 + lib/galaxy/jobs/__init__.py | 14 +- lib/galaxy/managers/histories.py | 4 +- lib/galaxy/managers/object_store_instances.py | 316 ++++++++++++ lib/galaxy/managers/users.py | 4 +- lib/galaxy/model/__init__.py | 54 +++ lib/galaxy/objectstore/__init__.py | 195 +++++++- lib/galaxy/objectstore/azure_blob.py | 1 + lib/galaxy/objectstore/badges.py | 26 +- lib/galaxy/objectstore/s3.py | 1 + lib/galaxy/objectstore/templates/__init__.py | 17 + lib/galaxy/objectstore/templates/manager.py | 141 ++++++ lib/galaxy/objectstore/templates/models.py | 249 ++++++++++ .../objectstore/unittest_utils/__init__.py | 13 +- lib/galaxy/security/validate_user_input.py | 11 +- lib/galaxy/security/vault.py | 22 + lib/galaxy/webapps/galaxy/api/object_store.py | 82 +++- lib/galaxy/webapps/galaxy/buildapp.py | 5 + lib/galaxy_test/base/populators.py | 36 +- lib/galaxy_test/driver/integration_util.py | 18 +- packages/objectstore/setup.cfg | 1 + pyproject.toml | 1 + test/integration/objectstore/test_per_user.py | 452 ++++++++++++++++++ ...ection_with_user_preferred_object_store.py | 5 +- .../test_from_configuration_object.py | 45 ++ .../test_serializing_user_object_stores.py | 64 +++ .../unit/objectstore/test_template_manager.py | 58 +++ test/unit/objectstore/test_template_models.py | 211 ++++++++ 71 files changed, 3770 insertions(+), 120 deletions(-) create mode 100644 client/src/components/Form/form-elements.scss create mode 100644 client/src/components/ObjectStore/Instances/CreateForm.test.ts create mode 100644 client/src/components/ObjectStore/Instances/CreateForm.vue create mode 100644 client/src/components/ObjectStore/Instances/CreateInstance.vue create mode 100644 client/src/components/ObjectStore/Instances/EditInstance.vue create mode 100644 client/src/components/ObjectStore/Instances/EditSecrets.vue create mode 100644 client/src/components/ObjectStore/Instances/InstanceDropdown.vue create mode 100644 client/src/components/ObjectStore/Instances/InstanceForm.vue create mode 100644 client/src/components/ObjectStore/Instances/ManageIndex.vue create mode 100644 client/src/components/ObjectStore/Instances/UpgradeForm.test.ts create mode 100644 client/src/components/ObjectStore/Instances/UpgradeForm.vue create mode 100644 client/src/components/ObjectStore/Instances/UpgradeInstance.vue create mode 100644 client/src/components/ObjectStore/Instances/VaultSecret.vue create mode 100644 client/src/components/ObjectStore/Instances/icons.ts create mode 100644 client/src/components/ObjectStore/Instances/instance.ts create mode 100644 client/src/components/ObjectStore/Instances/routing.ts create mode 100644 client/src/components/ObjectStore/Instances/services.ts create mode 100644 client/src/components/ObjectStore/Instances/types.ts create mode 100644 client/src/components/ObjectStore/Instances/util.ts create mode 100644 client/src/components/ObjectStore/ObjectStoreTypeSpan.vue create mode 100644 client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue create mode 100644 client/src/components/ObjectStore/Templates/SelectTemplate.vue create mode 100644 client/src/components/ObjectStore/Templates/TemplateSummary.vue create mode 100644 client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue create mode 100644 client/src/components/ObjectStore/Templates/TemplateSummarySpan.vue create mode 100644 client/src/components/ObjectStore/Templates/types.ts create mode 100644 client/src/components/ObjectStore/style.css create mode 100644 client/src/stores/objectStoreInstancesStore.test.ts create mode 100644 client/src/stores/objectStoreInstancesStore.ts create mode 100644 client/src/stores/objectStoreTemplatesStore.test.ts create mode 100644 client/src/stores/objectStoreTemplatesStore.ts create mode 100644 client/src/stores/testUtils.ts create mode 100644 lib/galaxy/managers/object_store_instances.py create mode 100644 lib/galaxy/objectstore/templates/__init__.py create mode 100644 lib/galaxy/objectstore/templates/manager.py create mode 100644 lib/galaxy/objectstore/templates/models.py create mode 100644 test/integration/objectstore/test_per_user.py create mode 100644 test/unit/objectstore/test_from_configuration_object.py create mode 100644 test/unit/objectstore/test_serializing_user_object_stores.py create mode 100644 test/unit/objectstore/test_template_manager.py create mode 100644 test/unit/objectstore/test_template_models.py diff --git a/client/src/api/objectStores.ts b/client/src/api/objectStores.ts index 6b1ace439818..0a9089e5df44 100644 --- a/client/src/api/objectStores.ts +++ b/client/src/api/objectStores.ts @@ -8,10 +8,20 @@ export async function getSelectableObjectStores() { } const getObjectStore = fetcher.path("/api/object_stores/{object_store_id}").method("get").create(); +const getUserObjectStoreInstance = fetcher + .path("/api/object_store_instances/{user_object_store_id}") + .method("get") + .create(); export async function getObjectStoreDetails(id: string) { - const { data } = await getObjectStore({ object_store_id: id }); - return data; + if (id.startsWith("user_objects://")) { + const userObjectStoreId = id.substring("user_objects://".length); + const { data } = await getUserObjectStoreInstance({ user_object_store_id: userObjectStoreId }); + return data; + } else { + const { data } = await getObjectStore({ object_store_id: id }); + return data; + } } const updateObjectStoreFetcher = fetcher.path("/api/datasets/{dataset_id}/object_store_id").method("put").create(); diff --git a/client/src/components/Form/FormElement.vue b/client/src/components/Form/FormElement.vue index 17314b77cbcc..5b6288570094 100644 --- a/client/src/components/Form/FormElement.vue +++ b/client/src/components/Form/FormElement.vue @@ -312,62 +312,5 @@ const isOptional = computed(() => !isRequired.value && attrs.value["optional"] ! diff --git a/client/src/components/Form/form-elements.scss b/client/src/components/Form/form-elements.scss new file mode 100644 index 000000000000..17aeb65b914e --- /dev/null +++ b/client/src/components/Form/form-elements.scss @@ -0,0 +1,58 @@ +@import "theme/blue.scss"; +@import "~@fortawesome/fontawesome-free/scss/_variables"; + +.ui-form-element { + margin-top: $margin-v * 0.25; + margin-bottom: $margin-v * 0.5; + overflow: visible; + clear: both; + + .ui-form-title { + word-wrap: break-word; + font-weight: bold; + + .ui-form-title-message { + font-size: $font-size-base * 0.7; + font-weight: 300; + vertical-align: text-top; + color: $text-light; + cursor: default; + } + + .ui-form-title-star { + color: $text-light; + font-weight: 300; + cursor: default; + } + + .warning { + color: $brand-danger; + } + } + + .ui-form-field { + position: relative; + margin-top: $margin-v * 0.25; + } + + &:deep(.ui-form-collapsible-icon), + &:deep(.ui-form-connected-icon) { + border: none; + background: none; + padding: 0; + line-height: 1; + font-size: 1.2em; + + &:hover { + color: $brand-info; + } + + &:focus { + color: $brand-primary; + } + + &:active { + background: none; + } + } +} diff --git a/client/src/components/ObjectStore/DescribeObjectStore.vue b/client/src/components/ObjectStore/DescribeObjectStore.vue index caf9ddd9ae45..1a9d6712ec3e 100644 --- a/client/src/components/ObjectStore/DescribeObjectStore.vue +++ b/client/src/components/ObjectStore/DescribeObjectStore.vue @@ -20,6 +20,7 @@ const props = defineProps(); const quotaSourceLabel = computed(() => props.storageInfo.quota?.source); const isPrivate = computed(() => props.storageInfo.private); const badges = computed(() => props.storageInfo.badges); +const userDefined = computed(() => props.storageInfo.object_store_id?.startsWith("user_objects://")); const quotaUsageProvider = ref(null); @@ -67,6 +68,9 @@ export default {
Galaxy has no quota configured for this object store.
- + diff --git a/client/src/components/ObjectStore/Instances/CreateForm.test.ts b/client/src/components/ObjectStore/Instances/CreateForm.test.ts new file mode 100644 index 000000000000..381817bc8d5f --- /dev/null +++ b/client/src/components/ObjectStore/Instances/CreateForm.test.ts @@ -0,0 +1,124 @@ +import { mount } from "@vue/test-utils"; +import flushPromises from "flush-promises"; +import { getLocalVue } from "tests/jest/helpers"; + +import { mockFetcher } from "@/api/schema/__mocks__"; +import type { ObjectStoreTemplateSummary } from "@/components/ObjectStore/Templates/types"; + +import CreateForm from "./CreateForm.vue"; + +jest.mock("@/api/schema"); + +const FAKE_OBJECT_STORE = "A fake object store"; + +const localVue = getLocalVue(true); + +const STANDARD_TEMPLATE: ObjectStoreTemplateSummary = { + type: "s3", + name: "moo", + description: null, + variables: [ + { + name: "myvar", + type: "string", + help: "*myvar help*", + }, + ], + secrets: [ + { + name: "mysecret", + help: "**mysecret help**", + }, + ], + id: "moo", + version: 0, + badges: [], +}; + +const SIMPLE_TEMPLATE: ObjectStoreTemplateSummary = { + type: "s3", + name: "moo", + description: null, + variables: [ + { + name: "myvar", + type: "string", + help: "*myvar help*", + }, + ], + secrets: [ + { + name: "mysecret", + help: "**mysecret help**", + }, + ], + id: "moo", + version: 0, + badges: [], +}; + +describe("CreateForm", () => { + it("should render a form with admin markdown converted to HTML in help", async () => { + const wrapper = mount(CreateForm, { + propsData: { + template: STANDARD_TEMPLATE, + }, + localVue, + }); + await flushPromises(); + + const varFormEl = wrapper.find("#form-element-myvar"); + expect(varFormEl).toBeTruthy(); + expect(varFormEl.html()).toContain("myvar help"); + + const secretFormEl = wrapper.find("#form-element-mysecret"); + expect(secretFormEl).toBeTruthy(); + expect(secretFormEl.html()).toContain("mysecret help"); + }); + + it("should post to create a new object store on submit", async () => { + const wrapper = mount(CreateForm, { + propsData: { + template: SIMPLE_TEMPLATE, + }, + localVue, + }); + mockFetcher.path("/api/object_store_instances").method("post").mock({ data: FAKE_OBJECT_STORE }); + await flushPromises(); + const nameForElement = wrapper.find("#form-element-_meta_name"); + nameForElement.find("input").setValue("My New Name"); + const submitElement = wrapper.find("#submit"); + submitElement.trigger("click"); + await flushPromises(); + const emitted = wrapper.emitted("created") || []; + expect(emitted).toHaveLength(1); + expect(emitted[0][0]).toBe(FAKE_OBJECT_STORE); + }); + + it("should indicate an error on failure", async () => { + const wrapper = mount(CreateForm, { + propsData: { + template: SIMPLE_TEMPLATE, + }, + localVue, + }); + mockFetcher + .path("/api/object_store_instances") + .method("post") + .mock(() => { + throw Error("Error creating this"); + }); + await flushPromises(); + const nameForElement = wrapper.find("#form-element-_meta_name"); + nameForElement.find("input").setValue("My New Name"); + const submitElement = wrapper.find("#submit"); + expect(wrapper.find(".object-store-instance-creation-error").exists()).toBe(false); + submitElement.trigger("click"); + await flushPromises(); + const emitted = wrapper.emitted("created") || []; + expect(emitted).toHaveLength(0); + const errorEl = wrapper.find(".object-store-instance-creation-error"); + expect(errorEl.exists()).toBe(true); + expect(errorEl.html()).toContain("Error creating this"); + }); +}); diff --git a/client/src/components/ObjectStore/Instances/CreateForm.vue b/client/src/components/ObjectStore/Instances/CreateForm.vue new file mode 100644 index 000000000000..7ff59ae6aebe --- /dev/null +++ b/client/src/components/ObjectStore/Instances/CreateForm.vue @@ -0,0 +1,81 @@ + + diff --git a/client/src/components/ObjectStore/Instances/CreateInstance.vue b/client/src/components/ObjectStore/Instances/CreateInstance.vue new file mode 100644 index 000000000000..91d2b996bb8e --- /dev/null +++ b/client/src/components/ObjectStore/Instances/CreateInstance.vue @@ -0,0 +1,34 @@ + + + diff --git a/client/src/components/ObjectStore/Instances/EditInstance.vue b/client/src/components/ObjectStore/Instances/EditInstance.vue new file mode 100644 index 000000000000..fc8bcbc5fecd --- /dev/null +++ b/client/src/components/ObjectStore/Instances/EditInstance.vue @@ -0,0 +1,90 @@ + + diff --git a/client/src/components/ObjectStore/Instances/EditSecrets.vue b/client/src/components/ObjectStore/Instances/EditSecrets.vue new file mode 100644 index 000000000000..da3bb94cc1b6 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/EditSecrets.vue @@ -0,0 +1,39 @@ + + diff --git a/client/src/components/ObjectStore/Instances/InstanceDropdown.vue b/client/src/components/ObjectStore/Instances/InstanceDropdown.vue new file mode 100644 index 000000000000..aaa2836c8289 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/InstanceDropdown.vue @@ -0,0 +1,57 @@ + + + diff --git a/client/src/components/ObjectStore/Instances/InstanceForm.vue b/client/src/components/ObjectStore/Instances/InstanceForm.vue new file mode 100644 index 000000000000..7c0131301671 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/InstanceForm.vue @@ -0,0 +1,43 @@ + + diff --git a/client/src/components/ObjectStore/Instances/ManageIndex.vue b/client/src/components/ObjectStore/Instances/ManageIndex.vue new file mode 100644 index 000000000000..9695f492dad2 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/ManageIndex.vue @@ -0,0 +1,106 @@ + + + diff --git a/client/src/components/ObjectStore/Instances/UpgradeForm.test.ts b/client/src/components/ObjectStore/Instances/UpgradeForm.test.ts new file mode 100644 index 000000000000..be8bfe4b78f1 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/UpgradeForm.test.ts @@ -0,0 +1,145 @@ +import { mount } from "@vue/test-utils"; +import flushPromises from "flush-promises"; +import { getLocalVue, injectTestRouter } from "tests/jest/helpers"; + +import { mockFetcher } from "@/api/schema/__mocks__"; +import type { ObjectStoreTemplateSummary } from "@/components/ObjectStore/Templates/types"; + +import type { UserConcreteObjectStore } from "./types"; + +import UpgradeForm from "./UpgradeForm.vue"; + +jest.mock("@/api/schema"); + +const localVue = getLocalVue(true); +const router = injectTestRouter(localVue); + +const STANDARD_TEMPLATE: ObjectStoreTemplateSummary = { + type: "s3", + name: "moo", + description: null, + variables: [ + { + name: "oldvar", + type: "string", + help: "old var help", + }, + { + name: "newvar", + type: "string", + help: "new var help", + }, + ], + secrets: [ + { + name: "oldsecret", + help: "old secret help", + }, + { + name: "newsecret", + help: "new secret help", + }, + ], + id: "moo", + version: 2, + badges: [], +}; + +const INSTANCE: UserConcreteObjectStore = { + type: "s3", + name: "moo", + description: undefined, + template_id: "moo", + template_version: 1, + badges: [], + variables: { + oldvar: "my old value", + droppedvar: "this will be dropped", + }, + secrets: ["oldsecret", "droppedsecret"], + quota: { enabled: false }, + private: false, + id: 4, +}; + +describe("UpgradeForm", () => { + it("should render with old variable values re-filled in", async () => { + const wrapper = mount(UpgradeForm, { + propsData: { + latestTemplate: STANDARD_TEMPLATE, + instance: INSTANCE, + }, + localVue, + router, + }); + await flushPromises(); + + const varFormEl = wrapper.find("#form-element-oldvar"); + expect(varFormEl).toBeTruthy(); + const inputField: HTMLInputElement = varFormEl.find("input").element as HTMLInputElement; + expect(inputField.value).toBe("my old value"); + }); + + it("should render with new variable values with empty values", async () => { + const wrapper = mount(UpgradeForm, { + propsData: { + latestTemplate: STANDARD_TEMPLATE, + instance: INSTANCE, + }, + localVue, + router, + }); + await flushPromises(); + + const varFormEl = wrapper.find("#form-element-newvar"); + expect(varFormEl).toBeTruthy(); + const inputField: HTMLInputElement = varFormEl.find("input").element as HTMLInputElement; + expect(inputField.value).toBe(""); + }); + + it("should put to update on submit and return to index", async () => { + const wrapper = mount(UpgradeForm, { + propsData: { + latestTemplate: STANDARD_TEMPLATE, + instance: INSTANCE, + }, + localVue, + router, + }); + mockFetcher.path("/api/object_store_instances/{user_object_store_id}").method("put").mock({ data: INSTANCE }); + await flushPromises(); + const submitElement = wrapper.find("#submit"); + submitElement.trigger("click"); + await flushPromises(); + const route = wrapper.vm.$route; + expect(route.path).toBe("/object_store_instances/index"); + expect(route.query.message).toBe("Upgraded object store moo"); + }); + + it("should indicate an error on failure", async () => { + const wrapper = mount(UpgradeForm, { + propsData: { + latestTemplate: STANDARD_TEMPLATE, + instance: INSTANCE, + }, + localVue, + router, + }); + mockFetcher + .path("/api/object_store_instances/{user_object_store_id}") + .method("put") + .mock(() => { + throw Error("problem upgrading"); + }); + await flushPromises(); + const submitElement = wrapper.find("#submit"); + expect(wrapper.find(".object-store-instance-upgrade-error").exists()).toBe(false); + submitElement.trigger("click"); + await flushPromises(); + const emitted = wrapper.emitted("created") || []; + expect(emitted).toHaveLength(0); + const errorEl = wrapper.find(".object-store-instance-upgrade-error"); + expect(errorEl.exists()).toBe(true); + expect(errorEl.html()).toContain("problem upgrading"); + }); +}); diff --git a/client/src/components/ObjectStore/Instances/UpgradeForm.vue b/client/src/components/ObjectStore/Instances/UpgradeForm.vue new file mode 100644 index 000000000000..d983b4a03d21 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/UpgradeForm.vue @@ -0,0 +1,86 @@ + + diff --git a/client/src/components/ObjectStore/Instances/UpgradeInstance.vue b/client/src/components/ObjectStore/Instances/UpgradeInstance.vue new file mode 100644 index 000000000000..df4fde6b2b6c --- /dev/null +++ b/client/src/components/ObjectStore/Instances/UpgradeInstance.vue @@ -0,0 +1,27 @@ + + diff --git a/client/src/components/ObjectStore/Instances/VaultSecret.vue b/client/src/components/ObjectStore/Instances/VaultSecret.vue new file mode 100644 index 000000000000..41fb485bd67d --- /dev/null +++ b/client/src/components/ObjectStore/Instances/VaultSecret.vue @@ -0,0 +1,65 @@ + + + + + diff --git a/client/src/components/ObjectStore/Instances/icons.ts b/client/src/components/ObjectStore/Instances/icons.ts new file mode 100644 index 000000000000..0c95a604c25c --- /dev/null +++ b/client/src/components/ObjectStore/Instances/icons.ts @@ -0,0 +1,5 @@ +import { library } from "@fortawesome/fontawesome-svg-core"; +import { faCaretDown, faPlus } from "@fortawesome/free-solid-svg-icons"; + +library.add(faCaretDown); +library.add(faPlus); diff --git a/client/src/components/ObjectStore/Instances/instance.ts b/client/src/components/ObjectStore/Instances/instance.ts new file mode 100644 index 000000000000..7ab30bfda7d7 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/instance.ts @@ -0,0 +1,25 @@ +import { computed, type Ref } from "vue"; + +import type { ObjectStoreTemplateSummary } from "@/components/ObjectStore/Templates/types"; +import { useObjectStoreInstancesStore } from "@/stores/objectStoreInstancesStore"; +import { useObjectStoreTemplatesStore } from "@/stores/objectStoreTemplatesStore"; + +import type { UserConcreteObjectStore } from "./types"; + +export function useInstanceAndTemplate(instanceIdRef: Ref) { + const objectStoreTemplatesStore = useObjectStoreTemplatesStore(); + const objectStoreInstancesStore = useObjectStoreInstancesStore(); + objectStoreInstancesStore.fetchInstances(); + objectStoreTemplatesStore.fetchTemplates(); + + const instance = computed( + () => objectStoreInstancesStore.getInstance(instanceIdRef.value) || null + ); + const template = computed(() => + instance.value + ? objectStoreTemplatesStore.getTemplate(instance.value?.template_id, instance.value?.template_version) + : null + ); + + return { instance, template }; +} diff --git a/client/src/components/ObjectStore/Instances/routing.ts b/client/src/components/ObjectStore/Instances/routing.ts new file mode 100644 index 000000000000..44aca677a052 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/routing.ts @@ -0,0 +1,16 @@ +import { useRouter } from "vue-router/composables"; + +export function useInstanceRouting() { + const router = useRouter(); + + async function goToIndex(query: Record<"message", string>) { + router.push({ + path: "/object_store_instances/index", + query: query, + }); + } + + return { + goToIndex, + }; +} diff --git a/client/src/components/ObjectStore/Instances/services.ts b/client/src/components/ObjectStore/Instances/services.ts new file mode 100644 index 000000000000..c61effa0e79b --- /dev/null +++ b/client/src/components/ObjectStore/Instances/services.ts @@ -0,0 +1,4 @@ +import { fetcher } from "@/api/schema/fetcher"; + +export const create = fetcher.path("/api/object_store_instances").method("post").create(); +export const update = fetcher.path("/api/object_store_instances/{user_object_store_id}").method("put").create(); diff --git a/client/src/components/ObjectStore/Instances/types.ts b/client/src/components/ObjectStore/Instances/types.ts new file mode 100644 index 000000000000..bd956468c7b8 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/types.ts @@ -0,0 +1,9 @@ +import type { components } from "@/api/schema/schema"; + +export type UserConcreteObjectStore = components["schemas"]["UserConcreteObjectStoreModel"]; +export type CreateInstancePayload = components["schemas"]["CreateInstancePayload"]; +export type ObjectStoreTemplateVariable = components["schemas"]["ObjectStoreTemplateVariable"]; +export type ObjectStoreTemplateSecret = components["schemas"]["ObjectStoreTemplateSecret"]; +export type VariableValueType = (string | boolean | number) | undefined; +export type VariableData = { [key: string]: VariableValueType }; +export type SecretData = { [key: string]: string }; diff --git a/client/src/components/ObjectStore/Instances/util.ts b/client/src/components/ObjectStore/Instances/util.ts new file mode 100644 index 000000000000..2ba02b8a75f2 --- /dev/null +++ b/client/src/components/ObjectStore/Instances/util.ts @@ -0,0 +1,45 @@ +import { markup } from "@/components/ObjectStore/configurationMarkdown"; + +import type { ObjectStoreTemplateSecret, ObjectStoreTemplateVariable, VariableValueType } from "./types"; + +export function metadataFormEntryName() { + return { + name: "_meta_name", + label: "Name", + type: "text", + optional: false, + help: "Label this new object store a name.", + }; +} + +export function metadataFormEntryDescription() { + return { + name: "_meta_description", + label: "Description", + optional: true, + type: "textarea", + help: "Provide some notes to yourself about this object store - perhaps to remind you how it is configured, where it stores the data, etc..", + }; +} + +export function templateVariableFormEntry(variable: ObjectStoreTemplateVariable, variableValue: VariableValueType) { + return { + name: variable.name, + type: "text", + help: markup(variable.help || "", true), + value: variableValue, + }; +} + +export function templateSecretFormEntry(secret: ObjectStoreTemplateSecret) { + return { + name: secret.name, + type: "password", + help: markup(secret.help || "", true), + value: "", + }; +} + +export function asNumber(x: number | string): number { + return +x; +} diff --git a/client/src/components/ObjectStore/ObjectStoreBadge.vue b/client/src/components/ObjectStore/ObjectStoreBadge.vue index 29f4db692934..530bd906b1e1 100644 --- a/client/src/components/ObjectStore/ObjectStoreBadge.vue +++ b/client/src/components/ObjectStore/ObjectStoreBadge.vue @@ -64,8 +64,8 @@ const shrink = computed(() => { return { transform: "shrink-6" }; }); -const message = computed(() => { - return props.badge.message; +const message = computed(() => { + return props.badge.message || ""; }); @@ -74,9 +74,7 @@ const message = computed(() => { - diff --git a/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.vue b/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.vue index aed05893e829..a7000878cbc9 100644 --- a/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.vue +++ b/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.vue @@ -19,13 +19,9 @@ const title = computed(() => { diff --git a/client/src/components/ObjectStore/ObjectStoreTypeSpan.vue b/client/src/components/ObjectStore/ObjectStoreTypeSpan.vue new file mode 100644 index 000000000000..6672e6a4523c --- /dev/null +++ b/client/src/components/ObjectStore/ObjectStoreTypeSpan.vue @@ -0,0 +1,27 @@ + + + + + diff --git a/client/src/components/ObjectStore/ShowSelectedObjectStore.test.js b/client/src/components/ObjectStore/ShowSelectedObjectStore.test.js index c20e3b861656..853a936548fa 100644 --- a/client/src/components/ObjectStore/ShowSelectedObjectStore.test.js +++ b/client/src/components/ObjectStore/ShowSelectedObjectStore.test.js @@ -13,12 +13,20 @@ jest.mock("@/api/schema"); const localVue = getLocalVue(true); const TEST_OBJECT_ID = "os123"; +const TEST_USER_OBJECT_STORE_ID = "user_objects://34"; + const OBJECT_STORE_DATA = { description: null, object_store_id: TEST_OBJECT_ID, badges: [], }; +const USER_OBJECT_STORE_DATA = { + description: null, + object_store_id: TEST_USER_OBJECT_STORE_ID, + badges: [], +}; + describe("ShowSelectedObjectStore", () => { let wrapper; @@ -36,4 +44,24 @@ describe("ShowSelectedObjectStore", () => { expect(loadingEl.exists()).toBeFalsy(); expect(wrapper.findComponent(DescribeObjectStore).exists()).toBeTruthy(); }); + + it("should fetch from the user based object store APIs for dynamic ids that are uris", async () => { + mockFetcher + .path("/api/object_store_instances/{user_object_store_id}") + .method("get") + .mock({ data: USER_OBJECT_STORE_DATA }); + // mockFetcher.path("/api/object_stores/{object_store_id}").method("get").mock({ data: OBJECT_STORE_DATA }); + + wrapper = mount(ShowSelectedObjectStore, { + propsData: { preferredObjectStoreId: TEST_USER_OBJECT_STORE_ID, forWhat: "Data goes into..." }, + localVue, + }); + let loadingEl = wrapper.findComponent(LoadingSpan); + expect(loadingEl.exists()).toBeTruthy(); + expect(loadingEl.find(".loading-message").text()).toContainLocalizationOf("Loading object store details"); + await flushPromises(); + loadingEl = wrapper.findComponent(LoadingSpan); + expect(loadingEl.exists()).toBeFalsy(); + expect(wrapper.findComponent(DescribeObjectStore).exists()).toBeTruthy(); + }); }); diff --git a/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue new file mode 100644 index 000000000000..10f5806d5db9 --- /dev/null +++ b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue @@ -0,0 +1,39 @@ + + diff --git a/client/src/components/ObjectStore/Templates/SelectTemplate.vue b/client/src/components/ObjectStore/Templates/SelectTemplate.vue new file mode 100644 index 000000000000..a4b1e2aebcc1 --- /dev/null +++ b/client/src/components/ObjectStore/Templates/SelectTemplate.vue @@ -0,0 +1,50 @@ + + + diff --git a/client/src/components/ObjectStore/Templates/TemplateSummary.vue b/client/src/components/ObjectStore/Templates/TemplateSummary.vue new file mode 100644 index 000000000000..c970cb707ac2 --- /dev/null +++ b/client/src/components/ObjectStore/Templates/TemplateSummary.vue @@ -0,0 +1,28 @@ + + + diff --git a/client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue b/client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue new file mode 100644 index 000000000000..868dca4ff83b --- /dev/null +++ b/client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue @@ -0,0 +1,21 @@ + + + diff --git a/client/src/components/ObjectStore/Templates/TemplateSummarySpan.vue b/client/src/components/ObjectStore/Templates/TemplateSummarySpan.vue new file mode 100644 index 000000000000..b52b31758670 --- /dev/null +++ b/client/src/components/ObjectStore/Templates/TemplateSummarySpan.vue @@ -0,0 +1,32 @@ + + + diff --git a/client/src/components/ObjectStore/Templates/types.ts b/client/src/components/ObjectStore/Templates/types.ts new file mode 100644 index 000000000000..186806b8e488 --- /dev/null +++ b/client/src/components/ObjectStore/Templates/types.ts @@ -0,0 +1,4 @@ +import type { components } from "@/api/schema/schema"; + +export type ObjectStoreTemplateSummary = components["schemas"]["ObjectStoreTemplateSummary"]; +export type ObjectStoreTemplateSummaries = ObjectStoreTemplateSummary[]; diff --git a/client/src/components/ObjectStore/style.css b/client/src/components/ObjectStore/style.css new file mode 100644 index 000000000000..c4dbdd000665 --- /dev/null +++ b/client/src/components/ObjectStore/style.css @@ -0,0 +1,4 @@ +.object-store-help-on-hover { + text-decoration-line: underline; + text-decoration-style: dashed; +} diff --git a/client/src/components/User/UserPreferences.vue b/client/src/components/User/UserPreferences.vue index b4c63a597076..d792e8720563 100644 --- a/client/src/components/User/UserPreferences.vue +++ b/client/src/components/User/UserPreferences.vue @@ -102,6 +102,14 @@ :preferred-object-store-id="currentUser.preferred_object_store_id" :user-id="userId"> + { + return { message: route.query["message"] }; + }, + }, + { + path: "object_store_instances/:instanceId/edit", + component: EditInstance, + props: true, + }, + { + path: "object_store_instances/:instanceId/upgrade", + component: UpgradeInstance, + props: true, + }, + { + path: "object_store_templates/:templateId/new", + component: CreateInstance, + props: true, + }, { path: "pages/create", component: FormGeneric, diff --git a/client/src/stores/objectStoreInstancesStore.test.ts b/client/src/stores/objectStoreInstancesStore.test.ts new file mode 100644 index 000000000000..659bd862c4fe --- /dev/null +++ b/client/src/stores/objectStoreInstancesStore.test.ts @@ -0,0 +1,57 @@ +import { useObjectStoreInstancesStore } from "@/stores/objectStoreInstancesStore"; + +import { setupTestPinia } from "./testUtils"; + +const type = "s3" as "s3" | "azure_blob" | "disk" | "generic_s3"; +const TEST_INSTANCE = { + type: type, + name: "moo", + description: undefined, + template_id: "an_s3_template", + template_version: 0, + badges: [], + variables: {}, + secrets: [], + quota: { enabled: false }, + private: false, + id: 4, +}; + +describe("Object Store Instances Store", () => { + beforeEach(setupTestPinia); + + it("should not be fetched initially", () => { + const objectStoreInstancesStore = useObjectStoreInstancesStore(); + expect(objectStoreInstancesStore.fetched).toBeFalsy(); + }); + + it("should not be in error initially", () => { + const objectStoreInstancesStore = useObjectStoreInstancesStore(); + expect(objectStoreInstancesStore.error).toBeFalsy(); + }); + + it("should populate store with handleInit", () => { + const objectStoreInstancesStore = useObjectStoreInstancesStore(); + objectStoreInstancesStore.handleInit([TEST_INSTANCE]); + expect(objectStoreInstancesStore.instances).toHaveLength(1); + expect(objectStoreInstancesStore.fetched).toBeTruthy(); + }); + + it("should allow finding an instance by instance id", () => { + const objectStoreInstancesStore = useObjectStoreInstancesStore(); + objectStoreInstancesStore.handleInit([TEST_INSTANCE]); + expect(objectStoreInstancesStore.getInstance(4)?.name).toBe("moo"); + }); + + it("should allow finding an instance by instance id as string (for props)", () => { + const objectStoreInstancesStore = useObjectStoreInstancesStore(); + objectStoreInstancesStore.handleInit([TEST_INSTANCE]); + expect(objectStoreInstancesStore.getInstance("4")?.name).toBe("moo"); + }); + + it("should populate an error with handleError", () => { + const objectStoreInstancesStore = useObjectStoreInstancesStore(); + objectStoreInstancesStore.handleError(Error("an error")); + expect(objectStoreInstancesStore.error).toBe("an error"); + }); +}); diff --git a/client/src/stores/objectStoreInstancesStore.ts b/client/src/stores/objectStoreInstancesStore.ts new file mode 100644 index 000000000000..38a410be6430 --- /dev/null +++ b/client/src/stores/objectStoreInstancesStore.ts @@ -0,0 +1,55 @@ +import { defineStore } from "pinia"; + +import { fetcher } from "@/api/schema/fetcher"; +import type { components } from "@/api/schema/schema"; +import { errorMessageAsString } from "@/utils/simple-error"; + +const getObjectStoreInstances = fetcher.path("/api/object_store_instances").method("get").create(); + +type UserConcreteObjectStoreModel = components["schemas"]["UserConcreteObjectStoreModel"]; + +function _asNumber(x: number | string): number { + return +x; +} + +export const useObjectStoreInstancesStore = defineStore("objectStoreInstances", { + state: () => ({ + instances: [] as UserConcreteObjectStoreModel[], + fetched: false, + error: null as string | null, + }), + getters: { + getInstances: (state) => { + return state.instances; + }, + loading: (state) => { + return !state.fetched; + }, + getInstance: (state) => { + return (id: number | string) => state.instances.find((i) => i.id == _asNumber(id)); + }, + }, + actions: { + async handleInit(instances: UserConcreteObjectStoreModel[]) { + this.instances = instances; + this.fetched = true; + this.error = null; + }, + async handleError(err: unknown) { + this.error = errorMessageAsString(err); + }, + async fetchInstances() { + try { + const { data: instances } = await getObjectStoreInstances({}); + this.handleInit(instances); + } catch (err) { + this.handleError(err); + } + }, + async ensureTemplates() { + if (!this.fetched || this.error != null) { + await this.fetchInstances(); + } + }, + }, +}); diff --git a/client/src/stores/objectStoreTemplatesStore.test.ts b/client/src/stores/objectStoreTemplatesStore.test.ts new file mode 100644 index 000000000000..2206712b212f --- /dev/null +++ b/client/src/stores/objectStoreTemplatesStore.test.ts @@ -0,0 +1,104 @@ +import { useObjectStoreTemplatesStore } from "@/stores/objectStoreTemplatesStore"; + +import { setupTestPinia } from "./testUtils"; + +const s3 = "s3" as "s3" | "azure_blob" | "disk" | "generic_s3"; +const TEMPLATES_BASIC = [ + { + type: s3, + name: "moo", + description: null, + variables: [], + secrets: [], + id: "moo", + version: 0, + badges: [], + }, +]; + +const TEMPLATES_EXPANDED = [ + { + type: s3, + name: "Testing S3", + description: null, + variables: [], + secrets: [], + id: "bucket_s3", + version: 0, + badges: [], + }, + { + type: s3, + name: "Testing S3 (some more)", + description: null, + variables: [], + secrets: [], + id: "bucket_s3", + version: 1, + badges: [], + }, + { + type: s3, + name: "Amazon S3 (working!)", + description: null, + variables: [], + secrets: [], + id: "bucket_s3", + version: 2, + badges: [], + }, +]; + +describe("Object Store Templates Store", () => { + beforeEach(setupTestPinia); + + it("should not be fetched initially", () => { + const objectStoreTemplateStore = useObjectStoreTemplatesStore(); + expect(objectStoreTemplateStore.fetched).toBeFalsy(); + }); + + it("should not be in error initially", () => { + const objectStoreTemplateStore = useObjectStoreTemplatesStore(); + expect(objectStoreTemplateStore.error).toBeFalsy(); + }); + + it("should populate store with handleInit", () => { + const objectStoreTemplateStore = useObjectStoreTemplatesStore(); + objectStoreTemplateStore.handleInit(TEMPLATES_BASIC); + expect(objectStoreTemplateStore.templates).toHaveLength(1); + expect(objectStoreTemplateStore.fetched).toBeTruthy(); + }); + + it("should find specific templates when multiple versions are available", () => { + const objectStoreTemplateStore = useObjectStoreTemplatesStore(); + objectStoreTemplateStore.handleInit(TEMPLATES_EXPANDED); + expect(objectStoreTemplateStore.templates).toHaveLength(3); + expect(objectStoreTemplateStore.fetched).toBeTruthy(); + const t1 = objectStoreTemplateStore.getTemplate("bucket_s3", 1); + expect(t1?.name).toBe("Testing S3 (some more)"); + const t2 = objectStoreTemplateStore.getTemplate("bucket_s3", 2); + expect(t2?.name).toBe("Amazon S3 (working!)"); + }); + + it("should define latest version getter that is collapsed", () => { + const objectStoreTemplateStore = useObjectStoreTemplatesStore(); + objectStoreTemplateStore.handleInit(TEMPLATES_EXPANDED); + expect(objectStoreTemplateStore.latestTemplates).toHaveLength(1); + const latestVersion = objectStoreTemplateStore.latestTemplates[0]; + expect(latestVersion?.name).toBe("Amazon S3 (working!)"); + }); + + it("should track what versions allow upgrade", () => { + const objectStoreTemplateStore = useObjectStoreTemplatesStore(); + objectStoreTemplateStore.handleInit(TEMPLATES_EXPANDED); + expect(objectStoreTemplateStore.canUpgrade("bucket_s3", 0)).toBeTruthy(); + expect(objectStoreTemplateStore.canUpgrade("bucket_s3", 1)).toBeTruthy(); + expect(objectStoreTemplateStore.canUpgrade("bucket_s3", 2)).toBeFalsy(); + }); + + it("should populate an error with handleError", () => { + const objectStoreTemplateStore = useObjectStoreTemplatesStore(); + objectStoreTemplateStore.handleError(Error("an error")); + expect(objectStoreTemplateStore.error).toBe("an error"); + }); +}); diff --git a/client/src/stores/objectStoreTemplatesStore.ts b/client/src/stores/objectStoreTemplatesStore.ts new file mode 100644 index 000000000000..c7c340a50f08 --- /dev/null +++ b/client/src/stores/objectStoreTemplatesStore.ts @@ -0,0 +1,115 @@ +import { defineStore } from "pinia"; + +import { fetcher } from "@/api/schema/fetcher"; +import type { components } from "@/api/schema/schema"; +import { errorMessageAsString } from "@/utils/simple-error"; + +const getObjectStoreTemplates = fetcher.path("/api/object_store_templates").method("get").create(); + +type ObjectStoreTemplateSummary = components["schemas"]["ObjectStoreTemplateSummary"]; +type ObjectStoreTemplateSummaries = ObjectStoreTemplateSummary[]; + +function findTemplate(templates: ObjectStoreTemplateSummaries, templateId: string, templateVersion: number) { + for (const template of templates) { + if (template.id == templateId && template.version == templateVersion) { + return template; + } + } + return null; +} + +function getLatestVersionMap(templates: ObjectStoreTemplateSummaries): { [key: string]: number } { + const latestVersions: { [key: string]: number } = {}; + templates.forEach((i: ObjectStoreTemplateSummary) => { + const templateId = i.id; + const templateVersion = i.version || 0; + if ((latestVersions[templateId] ?? -1) < templateVersion) { + latestVersions[templateId] = templateVersion; + } + }); + return latestVersions; +} + +function canUpgrade(templates: ObjectStoreTemplateSummaries, templateId: string, templateVersion: number): boolean { + let can = false; + templates.forEach((i: ObjectStoreTemplateSummary) => { + if (i.id == templateId && i.version && i.version > templateVersion) { + can = true; + } + }); + return can; +} + +function getLatestVersion(templates: ObjectStoreTemplateSummaries, id: string): ObjectStoreTemplateSummary | null { + let latestVersion = -1; + let latestTemplate = null as ObjectStoreTemplateSummary | null; + templates.forEach((i: ObjectStoreTemplateSummary) => { + const templateId = i.id; + if (templateId == id) { + const templateVersion = i.version || 0; + if (templateVersion > latestVersion) { + latestTemplate = i; + latestVersion = templateVersion; + } + } + }); + return latestTemplate; +} + +export const useObjectStoreTemplatesStore = defineStore("objectStoreTemplatesStore", { + state: () => ({ + templates: [] as ObjectStoreTemplateSummaries, + fetched: false, + error: null as string | null, + }), + getters: { + latestTemplates: (state) => { + // only expose latest instance by template_version for each template_id + const latestVersions = getLatestVersionMap(state.templates); + return state.templates.filter((i: ObjectStoreTemplateSummary) => latestVersions[i.id] == (i.version || 0)); + }, + canUpgrade: (state) => { + return (templateId: string, templateVersion: number) => + canUpgrade(state.templates, templateId, templateVersion); + }, + getTemplates: (state) => { + return state.templates; + }, + getTemplate: (state) => { + return (templateId: string, templateVersion: number) => + findTemplate(state.templates, templateId, templateVersion); + }, + getLatestTemplate: (state) => { + return (templateId: string) => getLatestVersion(state.templates, templateId); + }, + hasTemplates: (state) => { + return state.templates.length > 0; + }, + loading: (state) => { + return !state.fetched; + }, + }, + actions: { + async handleInit(templates: ObjectStoreTemplateSummaries) { + this.templates = templates; + this.fetched = true; + }, + async handleError(err: unknown) { + this.fetched = true; + this.error = errorMessageAsString(err); + }, + async fetchTemplates() { + try { + const { data: templates } = await getObjectStoreTemplates({}); + this.handleInit(templates); + } catch (err) { + this.handleError(err); + } + }, + async ensureTemplates() { + if (!this.fetched || this.error != null) { + await this.fetchTemplates(); + } + }, + }, +}); diff --git a/client/src/stores/testUtils.ts b/client/src/stores/testUtils.ts new file mode 100644 index 000000000000..906a724c7218 --- /dev/null +++ b/client/src/stores/testUtils.ts @@ -0,0 +1,5 @@ +import { createPinia, setActivePinia } from "pinia"; + +export function setupTestPinia() { + setActivePinia(createPinia()); +} diff --git a/client/src/utils/navigation/navigation.yml b/client/src/utils/navigation/navigation.yml index 88975e457fb6..937f616e82f3 100644 --- a/client/src/utils/navigation/navigation.yml +++ b/client/src/utils/navigation/navigation.yml @@ -97,12 +97,19 @@ preferences: email_input: "input[id='email']" username_input: "input[id='username']" preferred_storage: '.preferred-storage' + manage_object_stores: '.manage-object-stores' object_store_selection: selectors: option_buttons: '.preferred-object-store-select-button' option_button: '.preferred-object-store-select-button[data-object-store-id="${object_store_id}"]' +object_store_instances: + index: + selectors: + create_button: object-store-create + + toolbox_filters: selectors: input: diff --git a/client/tests/jest/helpers.js b/client/tests/jest/helpers.js index 06ce69efa015..ff5f1e716919 100644 --- a/client/tests/jest/helpers.js +++ b/client/tests/jest/helpers.js @@ -12,6 +12,7 @@ import { debounceTime, take, takeUntil } from "rxjs/operators"; import _l from "utils/localization"; import _short from "@/components/plugins/short"; +import VueRouter from "vue-router"; const defaultComparator = (a, b) => a == b; @@ -263,3 +264,12 @@ export function mockModule(storeModule, state = {}) { namespaced: true, }; } + +/** + * Return a new mocked out router attached the specified localVue instance. + */ +export function injectTestRouter(localVue) { + localVue.use(VueRouter); + const router = new VueRouter(); + return router; +} diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index f45c9aa83abc..cfecd43a6f67 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -50,6 +50,7 @@ from galaxy.managers.libraries import LibraryManager from galaxy.managers.library_datasets import LibraryDatasetsManager from galaxy.managers.notification import NotificationManager +from galaxy.managers.object_store_instances import UserObjectStoreResolverImpl from galaxy.managers.roles import RoleManager from galaxy.managers.session import GalaxySessionManager from galaxy.managers.tasks import ( @@ -91,7 +92,10 @@ from galaxy.objectstore import ( BaseObjectStore, build_object_store_from_config, + UserObjectStoreResolver, + UserObjectStoresAppConfig, ) +from galaxy.objectstore.templates import ConfiguredObjectStoreTemplates from galaxy.queue_worker import ( GalaxyQueueWorker, reload_toolbox, @@ -104,6 +108,7 @@ from galaxy.schema.fields import Security from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.vault import ( + is_vault_configured, Vault, VaultFactory, ) @@ -246,8 +251,6 @@ def __init__(self, fsmon=False, **kwargs) -> None: # Read config file and check for errors self.config = self._register_singleton(config.GalaxyAppConfiguration, config.GalaxyAppConfiguration(**kwargs)) self.config.check() - self._configure_object_store(fsmon=True) - self._register_singleton(BaseObjectStore, self.object_store) config_file = kwargs.get("global_conf", {}).get("__file__", None) if config_file: log.debug('Using "galaxy.ini" config file: %s', config_file) @@ -261,6 +264,10 @@ def __init__(self, fsmon=False, **kwargs) -> None: self._register_singleton(install_model_scoped_session, self.install_model.context) # Load quota management. self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model)) + self.vault = self._register_singleton(Vault, VaultFactory.from_app(self)) # type: ignore[type-abstract] + self._configure_object_store(fsmon=True) + self._register_singleton(BaseObjectStore, self.object_store) + galaxy.model.Dataset.object_store = self.object_store def configure_fluent_log(self): if self.config.fluent_log: @@ -405,6 +412,22 @@ def _configure_datatypes_registry(self, use_display_applications=True, use_conve ) def _configure_object_store(self, **kwds): + app_config = UserObjectStoresAppConfig( + jobs_directory=self.config.jobs_directory, + new_file_path=self.config.new_file_path, + umask=self.config.umask, + object_store_cache_size=self.config.object_store_cache_size, + object_store_cache_path=self.config.object_store_cache_path, + ) + self._register_singleton(UserObjectStoresAppConfig, app_config) + user_object_store_resolver = self._register_abstract_singleton( + UserObjectStoreResolver, UserObjectStoreResolverImpl # type: ignore[type-abstract] + ) # Ignored because of https://github.com/python/mypy/issues/4717 + vault_configured = is_vault_configured(self.vault) + templates = ConfiguredObjectStoreTemplates.from_app_config(self.config, vault_configured=vault_configured) + self.object_store_templates = self._register_singleton(ConfiguredObjectStoreTemplates, templates) + # kwds["object_store_templates"] = self.object_store_templates + kwds["user_object_store_resolver"] = user_object_store_resolver self.object_store = build_object_store_from_config(self.config, **kwds) def _configure_security(self): @@ -446,7 +469,7 @@ def _configure_models(self, check_migrate_databases=False, config_file=None): self.model = mapping.configure_model_mapping( self.config.file_path, - self.object_store, + None, # setting object store later now... self.config.use_pbkdf2, engine, combined_install_database, @@ -566,7 +589,6 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config) ) - self.vault = self._register_singleton(Vault, VaultFactory.from_app(self)) # type: ignore[type-abstract] # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.model.security.HostAgent( diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 823780a856f2..839dcfcecd36 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -551,6 +551,21 @@ mapping: desc: | FileSource plugins described embedded into Galaxy's config. + object_store_templates_config_file: + type: str + default: object_store_templates.yml + path_resolves_to: config_dir + required: false + desc: | + Configured Object Store templates configuration file. + + object_store_templates: + type: seq + sequence: + - type: any + desc: | + Configured Object Store templates embedded into Galaxy's config. + enable_mulled_containers: type: bool default: true diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index ae9a2dddca06..5228e4b0786b 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -68,7 +68,10 @@ ) from galaxy.model.base import transaction from galaxy.model.store.discover import MaxDiscoveredFilesExceededError -from galaxy.objectstore import ObjectStorePopulator +from galaxy.objectstore import ( + ObjectStorePopulator, + serialize_static_object_store_config, +) from galaxy.structured_app import MinimalManagerApp from galaxy.tool_util.deps import requirements from galaxy.tool_util.output_checker import ( @@ -2289,8 +2292,15 @@ def setup_external_metadata( self.app.datatypes_registry.to_xml_file(path=datatypes_config) inp_data, out_data, out_collections = job.io_dicts(exclude_implicit_outputs=True) + + required_user_object_store_uris = set() + for out_dataset_instance in out_data.values(): + object_store_id = out_dataset_instance.dataset.object_store_id + if object_store_id and object_store_id.startswith("user_objects://"): + required_user_object_store_uris.add(object_store_id) + job_metadata = os.path.join(self.tool_working_directory, self.tool.provided_metadata_file) - object_store_conf = self.object_store.to_dict() + object_store_conf = serialize_static_object_store_config(self.object_store, required_user_object_store_uris) command = self.external_output_metadata.setup_external_metadata( out_data, out_collections, diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index e704c3569815..a694d8775feb 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -992,7 +992,9 @@ def add_deserializers(self): def deserialize_preferred_object_store_id(self, item, key, val, **context): preferred_object_store_id = val - validation_error = validate_preferred_object_store_id(self.app.object_store, preferred_object_store_id) + validation_error = validate_preferred_object_store_id( + context["trans"], self.app.object_store, preferred_object_store_id + ) if validation_error: raise ModelDeserializingError(validation_error) return self.default_deserializer(item, key, preferred_object_store_id, **context) diff --git a/lib/galaxy/managers/object_store_instances.py b/lib/galaxy/managers/object_store_instances.py new file mode 100644 index 000000000000..fc5171df06f9 --- /dev/null +++ b/lib/galaxy/managers/object_store_instances.py @@ -0,0 +1,316 @@ +""" +To Test: +- upgrading with missing secret raise exception +- upgrading removes old secrets +- upgrading with partial variables keeps the old one +- upgrading with no variables works just fine +- upgrading and missing variables raises exception +""" + +import logging +from typing import ( + Dict, + List, + Optional, + Union, +) + +from pydantic import BaseModel + +from galaxy.exceptions import ( + ItemOwnershipException, + RequestParameterMissingException, +) +from galaxy.managers.context import ProvidesUserContext +from galaxy.model import ( + User, + UserObjectStore, +) +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.objectstore import ( + BaseUserObjectStoreResolver, + ConcreteObjectStoreModel, + QuotaModel, + UserObjectStoresAppConfig, +) +from galaxy.objectstore.badges import serialize_badges +from galaxy.objectstore.templates import ( + ConfiguredObjectStoreTemplates, + ObjectStoreConfiguration, + ObjectStoreTemplateSummaries, + ObjectStoreTemplateType, +) +from galaxy.objectstore.templates.models import ObjectStoreTemplateVariableValueType +from galaxy.security.vault import ( + UserVaultWrapper, + Vault, +) + +log = logging.getLogger(__name__) + + +class CreateInstancePayload(BaseModel): + name: str + description: Optional[str] = None + template_id: str + template_version: int + variables: Dict[str, ObjectStoreTemplateVariableValueType] + secrets: Dict[str, str] + + +class UpdateInstancePayload(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + variables: Optional[Dict[str, ObjectStoreTemplateVariableValueType]] + + +class UpdateInstanceSecretPayload(BaseModel): + secret_name: str + secret_value: str + + +class UpgradeInstancePayload(BaseModel): + template_version: int + variables: Dict[str, ObjectStoreTemplateVariableValueType] + secrets: Dict[str, str] + + +class UserConcreteObjectStoreModel(ConcreteObjectStoreModel): + id: int + type: ObjectStoreTemplateType + template_id: str + template_version: int + variables: Optional[Dict[str, ObjectStoreTemplateVariableValueType]] + secrets: List[str] + + +ModifyInstancePayload = Union[UpdateInstanceSecretPayload, UpgradeInstancePayload, UpdateInstancePayload] + + +class ObjectStoreInstancesManager: + _catalog: ConfiguredObjectStoreTemplates + _sa_session: galaxy_scoped_session + + def __init__( + self, + catalog: ConfiguredObjectStoreTemplates, + sa_session: galaxy_scoped_session, + vault: Vault, + ): + self._catalog = catalog + self._sa_session = sa_session + self._app_vault = vault + + @property + def summaries(self) -> ObjectStoreTemplateSummaries: + return self._catalog.summaries + + def modify_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: ModifyInstancePayload + ) -> UserConcreteObjectStoreModel: + if isinstance(payload, UpgradeInstancePayload): + return self._upgrade_instance(trans, id, payload) + elif isinstance(payload, UpdateInstanceSecretPayload): + return self._update_instance_secret(trans, id, payload) + else: + assert isinstance(payload, UpdateInstancePayload) + return self._update_instance(trans, id, payload) + + def _upgrade_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpgradeInstancePayload + ) -> UserConcreteObjectStoreModel: + persisted_object_store = self._get(trans, id) + catalog = self._catalog + template = catalog.find_template_by(persisted_object_store.object_store_template_id, payload.template_version) + persisted_object_store.object_store_template_version = template.version + persisted_object_store.object_store_template_definition = template.dict() + old_variables = persisted_object_store.object_store_template_variables + updated_variables = payload.variables + actual_variables = {} + for variable in template.variables or []: + variable_name = variable.name + old_value = old_variables.get(variable_name) + actual_variables[variable_name] = updated_variables.get(variable_name, old_value) + + persisted_object_store.object_store_template_variables = actual_variables + old_secrets = persisted_object_store.object_store_template_secrets + new_secrets = payload.secrets + + recorded_secrets = persisted_object_store.object_store_template_secrets + + user_vault = trans.user_vault + upgraded_template_secrets = [] + for secret in template.secrets or []: + secret_name = secret.name + upgraded_template_secrets.append(secret_name) + if secret_name not in new_secrets and secret_name not in old_secrets: + raise RequestParameterMissingException(f"secret {secret_name} not set in supplied request") + if secret_name not in new_secrets: + # keep old value + continue + + secret_value = new_secrets[secret_name] + key = user_vault_key(persisted_object_store, secret_name) + user_vault.write_secret(key, secret_value) + if secret_name not in recorded_secrets: + recorded_secrets.append(secret_name) + + for recorded_secret in recorded_secrets: + if recorded_secret not in upgraded_template_secrets: + key = user_vault_key(persisted_object_store, recorded_secret) + log.info(f"deleting {key} from user vault") + user_vault.delete_secret(key) + + persisted_object_store.object_store_template_secrets = recorded_secrets + self._save(persisted_object_store) + rval = self._to_model(trans, persisted_object_store) + return rval + + def _update_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpdateInstancePayload + ) -> UserConcreteObjectStoreModel: + # TODO: validate variables + # TODO: test case for access control + # TODO: test case for nulling update fields... + persisted_object_store = self._get(trans, id) + if payload.name is not None: + persisted_object_store.name = payload.name + if payload.description is not None: + persisted_object_store.description = payload.description + if payload.variables is not None: + # maybe just record the valid variables according to template like in upgrade + persisted_object_store.object_store_template_variables = payload.variables + self._save(persisted_object_store) + return self._to_model(trans, persisted_object_store) + + def _update_instance_secret( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpdateInstanceSecretPayload + ) -> UserConcreteObjectStoreModel: + persisted_object_store = self._get(trans, id) + user_vault = trans.user_vault + key = user_vault_key(persisted_object_store, payload.secret_name) + user_vault.write_secret(key, payload.secret_value) + return self._to_model(trans, persisted_object_store) + + def create_instance( + self, trans: ProvidesUserContext, payload: CreateInstancePayload + ) -> UserConcreteObjectStoreModel: + catalog = self._catalog + catalog.validate(payload) + template = catalog.find_template(payload) + assert template + user_vault = trans.user_vault + persisted_object_store = UserObjectStore() + persisted_object_store.user_id = trans.user.id + persisted_object_store.object_store_template_definition = template.dict() + persisted_object_store.object_store_template_id = template.id + persisted_object_store.object_store_template_version = template.version + persisted_object_store.object_store_template_variables = payload.variables + persisted_object_store.name = payload.name + persisted_object_store.description = payload.description + self._save(persisted_object_store) + + # the exception handling below will cleanup object stores that cannot be + # finalized with a successful secret setting but it might be worth considering + # something more robust. Two ideas would be to set a uuid on the persisted_object_store + # and key the secrets on that instead of the of the ID (but this raises the question + # are unused secrets in the vault maybe even worse than broken db objects) or + # set a state on the DB objects and with INITIAL and ACTIVE states. State + # idea might be nice because then we could add INACTIVE state that would prevent + # new data from being added but still allow access. + recorded_secrets = [] + try: + for secret, value in payload.secrets.items(): + key = user_vault_key(persisted_object_store, secret) + user_vault.write_secret(key, value) + recorded_secrets.append(secret) + except Exception: + self._sa_session.delete(persisted_object_store) + raise + persisted_object_store.object_store_template_secrets = recorded_secrets + self._save(persisted_object_store) + return self._to_model(trans, persisted_object_store) + + def index(self, trans: ProvidesUserContext) -> List[UserConcreteObjectStoreModel]: + stores = self._sa_session.query(UserObjectStore).filter(UserObjectStore.user_id == trans.user.id).all() + return [self._to_model(trans, s) for s in stores] + + def show(self, trans: ProvidesUserContext, id: Union[str, int]) -> UserConcreteObjectStoreModel: + user_object_store = self._get(trans, id) + return self._to_model(trans, user_object_store) + + def _save(self, persisted_object_store: UserObjectStore) -> None: + self._sa_session.add(persisted_object_store) + self._sa_session.flush([persisted_object_store]) + + def _get(self, trans: ProvidesUserContext, id: Union[str, int]) -> UserObjectStore: + user_object_store = self._sa_session.query(UserObjectStore).get(int(id)) + if user_object_store.user != trans.user: + raise ItemOwnershipException() + return user_object_store + + def _to_model(self, trans, persisted_object_store: UserObjectStore) -> UserConcreteObjectStoreModel: + quota = QuotaModel(source=None, enabled=False) + object_store_type = persisted_object_store.template.configuration.type + admin_badges = persisted_object_store.template.configuration.badges or [] + badges = serialize_badges( + admin_badges, + False, + True, + True, + object_store_type in ["azure_blob", "s3"], + ) + # These shouldn't be null but sometimes can be? + secrets = persisted_object_store.object_store_template_secrets or [] + return UserConcreteObjectStoreModel( + id=persisted_object_store.id, + type=object_store_type, + template_id=persisted_object_store.object_store_template_id, + template_version=persisted_object_store.object_store_template_version, + variables=persisted_object_store.object_store_template_variables, + secrets=secrets, + name=persisted_object_store.name, + description=persisted_object_store.description, + object_store_id=f"user_objects://{persisted_object_store.id}", + private=True, + quota=quota, + badges=badges, + ) + + +def user_vault_key(user_object_store: UserObjectStore, secret: str) -> str: + uos_id = user_object_store.id + assert uos_id + user_vault_id_prefix = f"object_store_config/{uos_id}" + key = f"{user_vault_id_prefix}/{secret}" + return key + + +def recover_secrets(user_object_store: UserObjectStore, vault: Vault) -> Dict[str, str]: + user: User = user_object_store.user + user_vault = UserVaultWrapper(vault, user) + secrets: Dict[str, str] = {} + # now we could recover the list of secrets to fetch from... + # ones recorded as written in the persisted object, the ones + # expected in the catalog, or the ones expected in the definition + # persisted. + for secret in user_object_store.object_store_template_secrets: + vault_key = user_vault_key(user_object_store, secret) + secret_value = user_vault.read_secret(vault_key) + assert secret_value + secrets[secret] = secret_value + return secrets + + +class UserObjectStoreResolverImpl(BaseUserObjectStoreResolver): + def __init__(self, sa_session: galaxy_scoped_session, vault: Vault, app_config: UserObjectStoresAppConfig): + self._sa_session = sa_session + self._vault = vault + self._app_config = app_config + + def resolve_object_store_uri_config(self, uri: str) -> ObjectStoreConfiguration: + user_object_store_id = uri.split("://", 1)[1] + user_object_store: UserObjectStore = self._sa_session.query(UserObjectStore).get(user_object_store_id) + secrets = recover_secrets(user_object_store, self._vault) + object_store_configuration = user_object_store.object_store_configuration(secrets=secrets) + return object_store_configuration diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 8f4b9c428e82..94e93110e5a3 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -799,9 +799,9 @@ def add_deserializers(self): } self.deserializers.update(user_deserializers) - def deserialize_preferred_object_store_id(self, item: Any, key: Any, val: Any, **context): + def deserialize_preferred_object_store_id(self, item: Any, key: Any, val: Any, trans=None, **context): preferred_object_store_id = val - validation_error = validate_preferred_object_store_id(self.app.object_store, preferred_object_store_id) + validation_error = validate_preferred_object_store_id(trans, self.app.object_store, preferred_object_store_id) if validation_error: raise base.ModelDeserializingError(validation_error) return self.default_deserializer(item, key, preferred_object_store_id, **context) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 77d518b60618..fc0f762398cf 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -141,6 +141,11 @@ from galaxy.model.orm.now import now from galaxy.model.orm.util import add_object_to_object_session from galaxy.objectstore import ObjectStore +from galaxy.objectstore.templates import ( + ObjectStoreConfiguration, + ObjectStoreTemplate, + template_to_configuration, +) from galaxy.schema.invocation import ( InvocationCancellationUserRequest, InvocationState, @@ -718,6 +723,7 @@ class User(Base, Dictifiable, RepresentById): galaxy_sessions = relationship( "GalaxySession", back_populates="user", order_by=lambda: desc(GalaxySession.update_time), cascade_backrefs=False # type: ignore[has-type] ) + object_stores = relationship("UserObjectStore", back_populates="user") quotas = relationship("UserQuotaAssociation", back_populates="user") quota_source_usages = relationship("UserQuotaSourceUsage", back_populates="user") social_auth = relationship("UserAuthnzToken", back_populates="user") @@ -10650,6 +10656,54 @@ def __init__(self, name=None, value=None): self.value = value +class UserObjectStore(Base, RepresentById): + __tablename__ = "user_object_store" + + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now, index=True) + # user specified name of the instance they've created + name = Column(String(255), index=True) + # user specified description of the instance they've created + description = Column(Text) + # the template store id + object_store_template_id = Column(String(255), index=True) + # the template store version (0, 1, ...) + object_store_template_version = Column(Integer, index=True) + # Full template from object_store_templates.yml catalog. + # For tools we just store references, so here we could easily just use + # the id/version and not record the definition... as the templates change + # over time this choice has some big consequences despite being easy to swap + # implementations. + object_store_template_definition = Column(JSONType) + # Big JSON blob of the variable name -> value mapping defined for the store's + # variables by the user. + object_store_template_variables = Column(JSONType) + # Track a list of secrets that were defined for this object store at creation + object_store_template_secrets = Column(JSONType) + + user = relationship("User", back_populates="object_stores") + + @property + def template(self) -> ObjectStoreTemplate: + return ObjectStoreTemplate(**self.object_store_template_definition) + + def object_store_configuration(self, secrets: Dict[str, Any]) -> ObjectStoreConfiguration: + user = self.user + user_details = { + "username": user.username, + "email": user.email, + "id": user.id, + } + return template_to_configuration( + self.template, + variables=self.object_store_template_variables, + secrets=secrets, + user_details=user_details, + ) + + class UserAction(Base, RepresentById): __tablename__ = "user_action" diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 13fed5fe0ddd..11825a26ca6e 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -18,6 +18,7 @@ List, NamedTuple, Optional, + Set, Tuple, Type, TYPE_CHECKING, @@ -25,8 +26,10 @@ import yaml from pydantic import BaseModel +from typing_extensions import Protocol from galaxy.exceptions import ( + ConfigDoesNotAllowException, MalformedContents, ObjectInvalid, ObjectNotFound, @@ -53,6 +56,7 @@ StoredBadgeDict, ) from .caching import CacheTarget +from .templates import ObjectStoreConfiguration if TYPE_CHECKING: from galaxy.model import DatasetInstance @@ -67,6 +71,27 @@ log = logging.getLogger(__name__) +class UserObjectStoreResolver(Protocol): + def resolve_object_store_uri_config(self, uri: str) -> ObjectStoreConfiguration: + pass + + def resolve_object_store_uri(self, uri: str) -> "ConcreteObjectStore": + pass + + +class BaseUserObjectStoreResolver(UserObjectStoreResolver, metaclass=abc.ABCMeta): + _app_config: "UserObjectStoresAppConfig" + + @abc.abstractmethod + def resolve_object_store_uri_config(self, uri: str) -> ObjectStoreConfiguration: + """Resolve the supplied object store URI into a concrete object store configuration.""" + pass + + def resolve_object_store_uri(self, uri: str) -> "ConcreteObjectStore": + object_store_configuration = self.resolve_object_store_uri_config(uri) + return concrete_object_store(object_store_configuration, self._app_config) + + class ObjectStore(metaclass=abc.ABCMeta): """ObjectStore interface. @@ -294,6 +319,11 @@ def object_store_allows_id_selection(self) -> bool: """Return True if this object store respects object_store_id and allow selection of this.""" return False + def validate_selected_object_store_id(self, user, object_store_id: Optional[str]) -> Optional[str]: + if object_store_id and not self.object_store_allows_id_selection(): + return "The current configuration doesn't allow selecting preferred object stores." + return None + def object_store_ids_allowing_selection(self) -> List[str]: """Return a non-emtpy list of allowed selectable object store IDs during creation.""" return [] @@ -508,6 +538,7 @@ class ConcreteObjectStore(BaseObjectStore): badges: List[StoredBadgeDict] device_id: Optional[str] = None + cloud: bool = False def __init__(self, config, config_dict=None, **kwargs): """ @@ -568,6 +599,8 @@ def _get_concrete_store_badges(self, obj) -> List[BadgeDict]: self.badges, self.galaxy_enable_quotas and self.quota_enabled, self.private, + False, + self.cloud, ) def _get_concrete_store_name(self, obj): @@ -794,7 +827,8 @@ def _exists(self, obj, **kwargs): # construct and check hashed path. if os.path.exists(path): return True - return os.path.exists(self._construct_path(obj, **kwargs)) + path = self._construct_path(obj, **kwargs) + return os.path.exists(path) def _create(self, obj, **kwargs): """Override `ObjectStore`'s stub by creating any files and folders on disk.""" @@ -1031,8 +1065,12 @@ class DistributedObjectStore(NestedObjectStore): """ store_type = "distributed" + _quota_source_map: Optional["QuotaSourceMap"] + _device_source_map: Optional["DeviceSourceMap"] - def __init__(self, config, config_dict, fsmon=False): + def __init__( + self, config, config_dict, fsmon=False, user_object_store_resolver: Optional[UserObjectStoreResolver] = None + ): """ :type config: object :param config: An object, most likely populated from @@ -1078,8 +1116,9 @@ def __init__(self, config, config_dict, fsmon=False): self.weighted_backend_ids.append(backened_id) self.original_weighted_backend_ids = self.weighted_backend_ids + self.user_object_store_resolver = user_object_store_resolver self.user_selection_allowed = user_selection_allowed - self.allow_user_selection = bool(user_selection_allowed) + self.allow_user_selection = bool(user_selection_allowed) or (user_object_store_resolver is not None) self.sleeper = None if fsmon and (self.global_max_percent_full or [_ for _ in self.max_percent_full.values() if _ != 0.0]): self.sleeper = Sleeper() @@ -1124,7 +1163,14 @@ def parse_xml(clazz, config_xml, legacy=False): return config_dict @classmethod - def from_xml(clazz, config, config_xml, fsmon=False): + def from_xml( + clazz, + config, + config_xml, + fsmon=False, + user_object_store_resolver: Optional[UserObjectStoreResolver] = None, + **kwd, + ): legacy = False if config_xml is None: distributed_config = config.distributed_object_store_config_file @@ -1141,9 +1187,9 @@ def from_xml(clazz, config, config_xml, fsmon=False): log.debug("Loading backends for distributed object store from %s", config_xml.get("id")) config_dict = clazz.parse_xml(config_xml, legacy=legacy) - return clazz(config, config_dict, fsmon=fsmon) + return clazz(config, config_dict, fsmon=fsmon, user_object_store_resolver=user_object_store_resolver) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self, object_store_uris: Optional[Set[str]] = None) -> Dict[str, Any]: as_dict = super().to_dict() as_dict["global_max_percent_full"] = self.global_max_percent_full as_dict["search_for_missing"] = self.search_for_missing @@ -1154,6 +1200,21 @@ def to_dict(self) -> Dict[str, Any]: backend_as_dict["max_percent_full"] = self.max_percent_full[backend_id] backend_as_dict["weight"] = len([i for i in self.original_weighted_backend_ids if i == backend_id]) backends.append(backend_as_dict) + if object_store_uris: + for user_object_store_uri in object_store_uris: + if not self.user_object_store_resolver: + raise ConfigDoesNotAllowException() + + object_store_config = self.user_object_store_resolver.resolve_object_store_uri_config( + user_object_store_uri + ) + dynamic_object_store_as_dict = object_store_config.model_dump() + dynamic_object_store_as_dict["id"] = user_object_store_uri + dynamic_object_store_as_dict["weight"] = 0 + # these are all forward facing object stores... + dynamic_object_store_as_dict["store_by"] = "uuid" + backends.append(dynamic_object_store_as_dict) + as_dict["backends"] = backends return as_dict @@ -1181,7 +1242,7 @@ def _create(self, obj, **kwargs): """The only method in which obj.object_store_id may be None.""" object_store_id = obj.object_store_id if object_store_id is None or not self._exists(obj, **kwargs): - if object_store_id is None or object_store_id not in self.backends: + if object_store_id is None or (object_store_id not in self.backends and "://" not in object_store_id): try: object_store_id = random.choice(self.weighted_backend_ids) obj.object_store_id = object_store_id @@ -1199,14 +1260,14 @@ def _create(self, obj, **kwargs): obj.__class__.__name__, obj.id, ) - return self.backends[object_store_id].create(obj, **kwargs) + return self._resolve_backend(object_store_id).create(obj, **kwargs) else: - return self.backends[object_store_id] + return self._resolve_backend(object_store_id) def _call_method(self, method, obj, default, default_is_exception, **kwargs): object_store_id = self.__get_store_id_for(obj, **kwargs) if object_store_id is not None: - return self.backends[object_store_id].__getattribute__(method)(obj, **kwargs) + return self._resolve_backend(object_store_id).__getattribute__(method)(obj, **kwargs) if default_is_exception: raise default( f"objectstore, _call_method failed: {method} on {self._repr_object_for_exception(obj)}, kwargs: {kwargs}" @@ -1214,11 +1275,20 @@ def _call_method(self, method, obj, default, default_is_exception, **kwargs): else: return default - def get_quota_source_map(self): + def _resolve_backend(self, object_store_id: str): + try: + return self.backends[object_store_id] + except KeyError: + if object_store_id.startswith("user_objects://") and self.user_object_store_resolver: + return self.user_object_store_resolver.resolve_object_store_uri(object_store_id) + raise + + def get_quota_source_map(self) -> "QuotaSourceMap": if self._quota_source_map is None: quota_source_map = QuotaSourceMap() self._merge_quota_source_map(quota_source_map, self) self._quota_source_map = quota_source_map + assert self._quota_source_map is not None return self._quota_source_map def get_device_source_map(self) -> "DeviceSourceMap": @@ -1226,6 +1296,7 @@ def get_device_source_map(self) -> "DeviceSourceMap": device_source_map = DeviceSourceMap() self._merge_device_source_map(device_source_map, self) self._device_source_map = device_source_map + assert self._device_source_map is not None return self._device_source_map @classmethod @@ -1246,7 +1317,7 @@ def _merge_device_source_map(clz, device_source_map: "DeviceSourceMap", object_s def __get_store_id_for(self, obj, **kwargs): if obj.object_store_id is not None: - if obj.object_store_id in self.backends: + if obj.object_store_id in self.backends or obj.object_store_id.startswith("user_objects://"): return obj.object_store_id else: log.warning( @@ -1280,8 +1351,26 @@ def object_store_allows_id_selection(self) -> bool: """Return True if this object store respects object_store_id and allow selection of this.""" return self.allow_user_selection + def validate_selected_object_store_id(self, user, object_store_id: Optional[str]) -> Optional[str]: + parent_check = super().validate_selected_object_store_id(user, object_store_id) + if parent_check or object_store_id is None: + return parent_check + # user selection allowed and object_store_id is not None + if object_store_id.startswith("user_objects://"): + if not user: + return "Supplied object store id is not accessible" + rest_of_uri = object_store_id.split("://", 1)[1] + user_object_store_id = int(rest_of_uri) + for user_object_store in user.object_stores: + if user_object_store.id == user_object_store_id: + return None + return "Supplied object store id was not found" + if object_store_id not in self.object_store_ids_allowing_selection(): + return "Supplied object store id is not an allowed object store selection" + return None + def object_store_ids_allowing_selection(self) -> List[str]: - """Return a non-emtpy list of allowed selectable object store IDs during creation.""" + """Return a non-empty list of allowed selectable object store IDs during creation.""" return self.user_selection_allowed def get_concrete_store_by_object_store_id(self, object_store_id: str) -> Optional["ConcreteObjectStore"]: @@ -1380,6 +1469,24 @@ def get_quota_source_map(self): return quota_source_map +def serialize_static_object_store_config(object_store: ObjectStore, object_store_uris: Set[str]) -> Dict[str, Any]: + """Serialize a static object store configuration for database-less serialization. + + The database-less part here comes from the fact these are used in job directories + during extended metadata collection. Any database/vault/app config details should + be unpacked and the result should be an object store configuration that doesn't + depend on those entities but which resolves to the same locations. + """ + if len(object_store_uris) == 0: + return object_store.to_dict() + if not isinstance(object_store, DistributedObjectStore): + # TODO: Not for the MVP or first iteration - but potentially we could allow + # a concrete store here and then build a Distributed store from that and + # concrete stores represented by object_store_uris + raise ConfigDoesNotAllowException("ObjectStore configuration does not allow per-user object stores") + return object_store.to_dict(object_store_uris=object_store_uris) + + class QuotaModel(BaseModel): source: Optional[str] = None enabled: bool @@ -1395,9 +1502,11 @@ class ConcreteObjectStoreModel(BaseModel): device: Optional[str] = None -def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[BaseObjectStore], Dict[str, Any]]: +def type_to_object_store_class( + store: str, fsmon: bool = False, user_object_store_resolver: Optional[UserObjectStoreResolver] = None +) -> Tuple[Type[BaseObjectStore], Dict[str, Any]]: objectstore_class: Type[BaseObjectStore] - objectstore_constructor_kwds = {} + objectstore_constructor_kwds: Dict[str, Any] = {} if store == "disk": objectstore_class = DiskObjectStore elif store in ["s3", "aws_s3"]: @@ -1415,6 +1524,7 @@ def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[Ba elif store == "distributed": objectstore_class = DistributedObjectStore objectstore_constructor_kwds["fsmon"] = fsmon + objectstore_constructor_kwds["user_object_store_resolver"] = user_object_store_resolver elif store == "hierarchical": objectstore_class = HierarchicalObjectStore objectstore_constructor_kwds["fsmon"] = fsmon @@ -1441,7 +1551,12 @@ def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[Ba def build_object_store_from_config( - config, fsmon=False, config_xml=None, config_dict=None, disable_process_management=False + config, + fsmon=False, + config_xml=None, + config_dict=None, + disable_process_management=False, + user_object_store_resolver: Optional[UserObjectStoreResolver] = None, ): """ Invoke the appropriate object store. @@ -1491,13 +1606,59 @@ def build_object_store_from_config( from_object = "dict" store = config_dict.get("type") - objectstore_class, objectstore_constructor_kwds = type_to_object_store_class(store, fsmon=fsmon) + objectstore_class, objectstore_constructor_kwds = type_to_object_store_class( + store, fsmon=fsmon, user_object_store_resolver=user_object_store_resolver + ) if from_object == "xml": return objectstore_class.from_xml(config=config, config_xml=config_xml, **objectstore_constructor_kwds) else: return objectstore_class(config=config, config_dict=config_dict, **objectstore_constructor_kwds) +# View into the application configuration that is shared between the global object store +# and user defined object stores as produced by concrete_object_store. +class UserObjectStoresAppConfig(BaseModel): + object_store_cache_path: str + object_store_cache_size: int + jobs_directory: str + new_file_path: str + umask: int + + +# TODO: this will need app details... +# TODO: unit test from configuration dict... +def concrete_object_store( + object_store_configuration: ObjectStoreConfiguration, app_config: UserObjectStoresAppConfig +) -> ConcreteObjectStore: + # Adapt structured UserObjectStoresAppConfig into a more full configuration object as expected by + # the object stores + class GalaxyConfigAdapter: + # Hard code these, these will not support legacy features + object_store_check_old_style = False + object_store_store_by = "uuid" + + # Set this to false for now... not sure but we may want to revisit this + enable_quotas = False + + # These need to come in from Galaxy's config + jobs_directory = app_config.jobs_directory + new_file_path = app_config.new_file_path + umask = app_config.umask + object_store_cache_size = app_config.object_store_cache_size + object_store_cache_path = app_config.object_store_cache_path + + objectstore_class, objectstore_constructor_kwds = type_to_object_store_class( + store=object_store_configuration.type, + fsmon=False, + ) + assert issubclass(objectstore_class, ConcreteObjectStore) + return objectstore_class( + config=GalaxyConfigAdapter(), + config_dict=object_store_configuration.model_dump(), + **objectstore_constructor_kwds, + ) + + def local_extra_dirs(func): """Non-local plugin decorator using local directories for the extra_dirs (job_work and temp).""" diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py index 83c1d700e195..7302fc9fa0bd 100644 --- a/lib/galaxy/objectstore/azure_blob.py +++ b/lib/galaxy/objectstore/azure_blob.py @@ -90,6 +90,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): cache_monitor: Optional[InProcessCacheMonitor] = None store_type = "azure_blob" + cloud = True def __init__(self, config, config_dict): super().__init__(config, config_dict) diff --git a/lib/galaxy/objectstore/badges.py b/lib/galaxy/objectstore/badges.py index 9f0a61adcd2a..5d1fae30d906 100644 --- a/lib/galaxy/objectstore/badges.py +++ b/lib/galaxy/objectstore/badges.py @@ -9,6 +9,7 @@ from typing_extensions import ( Literal, + NotRequired, TypedDict, ) @@ -34,6 +35,7 @@ "quota", "no_quota", "restricted", + "user_defined", ], ] @@ -69,11 +71,11 @@ class BadgeDict(TypedDict): class StoredBadgeDict(TypedDict): type: AdminBadgeT - message: Optional[str] + message: NotRequired[Optional[str]] def read_badges(config_dict: Dict[str, Any]) -> List[StoredBadgeDict]: - raw_badges = config_dict.get("badges", []) + raw_badges = config_dict.get("badges") or [] badges: List[StoredBadgeDict] = [] badge_types: Set[str] = set() badge_conflicts: Dict[str, str] = {} @@ -101,7 +103,9 @@ def read_badges(config_dict: Dict[str, Any]) -> List[StoredBadgeDict]: return badges -def serialize_badges(stored_badges: List[StoredBadgeDict], quota_enabled: bool, private: bool) -> List[BadgeDict]: +def serialize_badges( + stored_badges: List[StoredBadgeDict], quota_enabled: bool, private: bool, user_defined: bool, cloud: bool +) -> List[BadgeDict]: """Produce blended, unified list of badges for target object store entity. Combine more free form admin information stored about badges with Galaxy tracked @@ -113,7 +117,7 @@ def serialize_badges(stored_badges: List[StoredBadgeDict], quota_enabled: bool, badge_dict: BadgeDict = { "source": "admin", "type": badge["type"], - "message": badge["message"], + "message": badge.get("message"), } badge_dicts.append(badge_dict) @@ -138,4 +142,18 @@ def serialize_badges(stored_badges: List[StoredBadgeDict], quota_enabled: bool, "source": "galaxy", } badge_dicts.append(restricted_badge_dict) + if user_defined: + user_defined_badge_dict: BadgeDict = { + "type": "user_defined", + "message": None, + "source": "galaxy", + } + badge_dicts.append(user_defined_badge_dict) + if cloud: + cloud_badge_dict: BadgeDict = { + "type": "cloud", + "message": None, + "source": "galaxy", + } + badge_dicts.append(cloud_badge_dict) return badge_dicts diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 7e17c34d844e..43dce2ea55df 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -160,6 +160,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): cache_monitor: Optional[InProcessCacheMonitor] = None store_type = "aws_s3" + cloud = True def __init__(self, config, config_dict): super().__init__(config, config_dict) diff --git a/lib/galaxy/objectstore/templates/__init__.py b/lib/galaxy/objectstore/templates/__init__.py new file mode 100644 index 000000000000..7c7208dd7f04 --- /dev/null +++ b/lib/galaxy/objectstore/templates/__init__.py @@ -0,0 +1,17 @@ +from .manager import ConfiguredObjectStoreTemplates +from .models import ( + ObjectStoreConfiguration, + ObjectStoreTemplate, + ObjectStoreTemplateSummaries, + ObjectStoreTemplateType, + template_to_configuration, +) + +__all__ = ( + "ConfiguredObjectStoreTemplates", + "ObjectStoreConfiguration", + "ObjectStoreTemplate", + "ObjectStoreTemplateSummaries", + "ObjectStoreTemplateType", + "template_to_configuration", +) diff --git a/lib/galaxy/objectstore/templates/manager.py b/lib/galaxy/objectstore/templates/manager.py new file mode 100644 index 000000000000..901931b956c2 --- /dev/null +++ b/lib/galaxy/objectstore/templates/manager.py @@ -0,0 +1,141 @@ +import os +from typing import ( + Any, + Dict, + List, + Optional, +) + +from typing_extensions import Protocol +from yaml import safe_load + +from galaxy.exceptions import ( + ObjectNotFound, + RequestParameterMissingException, +) +from galaxy.objectstore.badges import serialize_badges +from .models import ( + ObjectStoreTemplate, + ObjectStoreTemplateCatalog, + ObjectStoreTemplateSummaries, +) + +RawTemplateConfig = Dict[str, Any] + + +class AppConfigProtocol(Protocol): + object_store_templates: Optional[List[RawTemplateConfig]] + object_store_templates_config_file: Optional[str] + + +class TemplateReference(Protocol): + template_id: str + template_version: int + + +class InstanceDefinition(TemplateReference, Protocol): + variables: Dict[str, Any] + secrets: Dict[str, str] + + +SECRETS_NEED_VAULT_MESSAGE = "The object store templates configuration can not be used - a Galaxy vault must be configured for templates that use secrets - please set the vault_config_file configuration option to point at a valid vault configuration." + + +class ConfiguredObjectStoreTemplates: + catalog: ObjectStoreTemplateCatalog + + def __init__(self, catalog: ObjectStoreTemplateCatalog): + self.catalog = catalog + + @staticmethod + def from_app_config(config: AppConfigProtocol, vault_configured=None) -> "ConfiguredObjectStoreTemplates": + raw_config = config.object_store_templates + if raw_config is None: + config_file = config.object_store_templates_config_file + if config_file and os.path.exists(config_file): + with open(config_file) as f: + raw_config = safe_load(f) + if raw_config is None: + raw_config = [] + templates = ConfiguredObjectStoreTemplates(raw_config_to_catalog(raw_config)) + if vault_configured is False and templates.configuration_uses_secrets: + raise Exception(SECRETS_NEED_VAULT_MESSAGE) + return templates + + @property + def configuration_uses_secrets(self) -> bool: + templates = self.catalog.root + for template in templates: + if template.secrets and len(template.secrets) > 0: + return True + return False + + @property + def summaries(self) -> ObjectStoreTemplateSummaries: + templates = self.catalog.root + summaries = [] + for template in templates: + template_dict = template.model_dump() + configuration = template_dict.pop("configuration") + stored_badges = configuration["badges"] or [] + object_store_type = configuration["type"] + badges = serialize_badges(stored_badges, False, True, True, object_store_type in ["azure", "s3"]) + template_dict["badges"] = badges + template_dict["type"] = object_store_type + summaries.append(template_dict) + return ObjectStoreTemplateSummaries.model_validate(summaries) + + def find_template(self, instance_reference: TemplateReference) -> ObjectStoreTemplate: + """Find the corresponding template and throw ObjectNotFound if not available.""" + template_id = instance_reference.template_id + template_version = instance_reference.template_version + return self.find_template_by(template_id, template_version) + + def find_template_by(self, template_id: str, template_version: int) -> ObjectStoreTemplate: + templates = self.catalog.root + + for template in templates: + if template.id == template_id and template.version == template_version: + return template + + raise ObjectNotFound( + f"Could not find a object store template with id {template_id} and version {template_version}" + ) + + def validate(self, instance: InstanceDefinition): + template = self.find_template(instance) + secrets = instance.secrets + for template_secret in template.secrets or []: + name = template_secret.name + if name not in secrets: + raise RequestParameterMissingException(f"Must define secret '{name}'") + variables = instance.variables + for template_variable in template.variables or []: + name = template_variable.name + if name not in variables: + raise RequestParameterMissingException(f"Must define variable '{name}'") + # TODO: validate no extra variables + + +def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> ObjectStoreTemplateCatalog: + effective_root = _apply_syntactic_sugar(raw_config) + return ObjectStoreTemplateCatalog.model_validate(effective_root) + + +def _apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]: + templates = [] + for template in raw_templates: + _force_key_to_list(template, "variables") + _force_key_to_list(template, "secrets") + templates.append(template) + return templates + + +def _force_key_to_list(template: RawTemplateConfig, key: str) -> None: + value = template.get(key, None) + if isinstance(value, dict): + value_as_list = [] + for key_name, key_value in value.items(): + key_value["name"] = key_name + value_as_list.append(key_value) + template[key] = value_as_list diff --git a/lib/galaxy/objectstore/templates/models.py b/lib/galaxy/objectstore/templates/models.py new file mode 100644 index 000000000000..9f225786194a --- /dev/null +++ b/lib/galaxy/objectstore/templates/models.py @@ -0,0 +1,249 @@ +from typing import ( + Any, + Dict, + List, + Optional, + Type, + Union, +) + +from boltons.iterutils import remap +from jinja2.nativetypes import NativeEnvironment +from pydantic import ( + BaseModel, + ConfigDict, + RootModel, +) +from typing_extensions import Literal + +from galaxy.objectstore.badges import ( + BadgeDict, + StoredBadgeDict, +) + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +ObjectStoreTemplateVariableType = Literal["string", "boolean", "integer"] +ObjectStoreTemplateVariableValueType = Union[str, bool, int] +TemplateExpansion = str +ObjectStoreTemplateType = Literal["s3", "azure_blob", "disk", "generic_s3"] + + +class S3AuthTemplate(StrictModel): + access_key: Union[str, TemplateExpansion] + secret_key: Union[str, TemplateExpansion] + + +class S3Auth(StrictModel): + access_key: str + secret_key: str + + +class S3BucketTemplate(StrictModel): + name: Union[str, TemplateExpansion] + use_reduced_redundancy: Optional[Union[bool, TemplateExpansion]] = None + + +class S3Bucket(StrictModel): + name: str + use_reduced_redundancy: Optional[bool] = None + + +BadgeList = Optional[List[StoredBadgeDict]] + + +class S3ObjectStoreTemplateConfiguration(StrictModel): + type: Literal["s3"] + auth: S3AuthTemplate + bucket: S3BucketTemplate + badges: BadgeList = None + + +class S3ObjectStoreConfiguration(StrictModel): + type: Literal["s3"] + auth: S3Auth + bucket: S3Bucket + badges: BadgeList = None + + +class AzureAuthTemplate(StrictModel): + account_name: Union[str, TemplateExpansion] + account_key: Union[str, TemplateExpansion] + + +class AzureAuth(StrictModel): + account_name: str + account_key: str + + +class AzureContainerTemplate(StrictModel): + name: Union[str, TemplateExpansion] + + +class AzureContainer(StrictModel): + name: str + + +class AzureObjectStoreTemplateConfiguration(StrictModel): + type: Literal["azure_blob"] + auth: AzureAuthTemplate + container: AzureContainerTemplate + badges: BadgeList = None + + +class AzureObjectStoreConfiguration(StrictModel): + type: Literal["azure_blob"] + auth: AzureAuth + container: AzureContainer + badges: BadgeList = None + + +class DiskObjectStoreTemplateConfiguration(StrictModel): + type: Literal["disk"] + files_dir: Union[str, TemplateExpansion] + badges: BadgeList = None + + +class DiskObjectStoreConfiguration(StrictModel): + type: Literal["disk"] + files_dir: str + badges: BadgeList = None + + +class S3ConnectionTemplate(StrictModel): + host: Union[str, TemplateExpansion] + port: Union[int, TemplateExpansion] + is_secure: Optional[Union[bool, TemplateExpansion]] = True + conn_path: Optional[Union[str, TemplateExpansion]] = "" + + +class S3Connection(StrictModel): + host: str + port: int + is_secure: bool = True + conn_path: str = "" + + +class GenericS3ObjectStoreTemplateConfiguration(StrictModel): + type: Literal["generic_s3"] + auth: S3AuthTemplate + bucket: S3BucketTemplate + connection: S3ConnectionTemplate + badges: BadgeList = None + + +class GenericS3ObjectStoreConfiguration(StrictModel): + type: Literal["generic_s3"] + auth: S3Auth + bucket: S3Bucket + connection: S3Connection + badges: BadgeList = None + + +ObjectStoreTemplateConfiguration = Union[ + S3ObjectStoreTemplateConfiguration, + GenericS3ObjectStoreTemplateConfiguration, + DiskObjectStoreTemplateConfiguration, + AzureObjectStoreTemplateConfiguration, +] +ObjectStoreConfiguration = Union[ + S3ObjectStoreConfiguration, + DiskObjectStoreConfiguration, + AzureObjectStoreConfiguration, + GenericS3ObjectStoreConfiguration, +] +MarkdownContent = str + + +class ObjectStoreTemplateVariable(StrictModel): + name: str + help: Optional[MarkdownContent] + type: ObjectStoreTemplateVariableType + + +class ObjectStoreTemplateSecret(StrictModel): + name: str + help: Optional[MarkdownContent] + + +class ObjectStoreTemplateBase(StrictModel): + """Version of ObjectStoreTemplate we can send to the UI/API. + + The configuration key in the child type may have secretes + and shouldn't be exposed over the API - at least to non-admins. + """ + + id: str + name: Optional[str] + description: Optional[MarkdownContent] + # The UI should just show the most recent version but allow + # admins to define newer versions with new parameterizations + # and keep old versions in template catalog for backward compatibility + # for users with existing stores of that template. + version: int = 0 + # Like with multiple versions, allow admins to deprecate a + # template by hiding but keep it in the catalog for backward + # compatibility for users with existing stores of that template. + hidden: bool = False + variables: Optional[List[ObjectStoreTemplateVariable]] = None + secrets: Optional[List[ObjectStoreTemplateSecret]] = None + + +class ObjectStoreTemplateSummary(ObjectStoreTemplateBase): + badges: List[BadgeDict] + type: ObjectStoreTemplateType + + +class ObjectStoreTemplate(ObjectStoreTemplateBase): + configuration: ObjectStoreTemplateConfiguration + + +ObjectStoreTemplateCatalog = RootModel[List[ObjectStoreTemplate]] + + +class ObjectStoreTemplateSummaries(RootModel): + root: List[ObjectStoreTemplateSummary] + + +def template_to_configuration( + template: ObjectStoreTemplate, + variables: Dict[str, ObjectStoreTemplateVariableValueType], + secrets: Dict[str, str], + user_details: Dict[str, Any], +) -> ObjectStoreConfiguration: + configuration_template = template.configuration + template_variables = { + "variables": variables, + "secrets": secrets, + "user": user_details, + } + + def expand_template(_, key, value): + if isinstance(value, str) and "{{" in value and "}}" in value: + # NativeEnvironment preserves Python types + template = NativeEnvironment().from_string(value) + return key, template.render(**template_variables) + return key, value + + raw_config = remap(configuration_template.model_dump(), visit=expand_template) + return to_configuration_object(raw_config) + + +TypesToConfigurationClasses: Dict[ObjectStoreTemplateType, Type[ObjectStoreConfiguration]] = { + "s3": S3ObjectStoreConfiguration, + "generic_s3": GenericS3ObjectStoreConfiguration, + "azure_blob": AzureObjectStoreConfiguration, + "disk": DiskObjectStoreConfiguration, +} + + +def to_configuration_object(configuration_dict: Dict[str, Any]) -> ObjectStoreConfiguration: + if "type" not in configuration_dict: + raise KeyError("Configuration objects require an object store 'type' key, none found.") + object_store_type = configuration_dict["type"] + if object_store_type not in TypesToConfigurationClasses: + raise ValueError(f"Unknown object store type found in raw configuration dictionary ({object_store_type}).") + return TypesToConfigurationClasses[object_store_type](**configuration_dict) diff --git a/lib/galaxy/objectstore/unittest_utils/__init__.py b/lib/galaxy/objectstore/unittest_utils/__init__.py index f96b12d91a0c..31e0964cd14a 100644 --- a/lib/galaxy/objectstore/unittest_utils/__init__.py +++ b/lib/galaxy/objectstore/unittest_utils/__init__.py @@ -5,6 +5,7 @@ from shutil import rmtree from string import Template from tempfile import mkdtemp +from typing import Optional import yaml @@ -32,7 +33,13 @@ class Config: - def __init__(self, config_str=DISK_TEST_CONFIG, clazz=None, store_by="id"): + def __init__( + self, + config_str=DISK_TEST_CONFIG, + clazz=None, + store_by="id", + user_object_store_resolver: Optional[objectstore.UserObjectStoreResolver] = None, + ): self.temp_directory = mkdtemp() if config_str.startswith("<"): config_file = "store.xml" @@ -42,7 +49,9 @@ def __init__(self, config_str=DISK_TEST_CONFIG, clazz=None, store_by="id"): config = MockConfig(self.temp_directory, config_file, store_by=store_by) self.global_config = config if clazz is None: - self.object_store = objectstore.build_object_store_from_config(config) + self.object_store = objectstore.build_object_store_from_config( + config, user_object_store_resolver=user_object_store_resolver + ) elif config_file == "store.xml": self.object_store = clazz.from_xml(config, XML(config_str)) else: diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index 5e21c86c6c4c..9e6ed7d4afc3 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -160,10 +160,7 @@ def validate_password(trans, password, confirm): return validate_password_str(password) -def validate_preferred_object_store_id(object_store: ObjectStore, preferred_object_store_id: Optional[str]) -> str: - if not object_store.object_store_allows_id_selection() and preferred_object_store_id is not None: - return "The current configuration doesn't allow selecting preferred object stores." - if object_store.object_store_allows_id_selection() and preferred_object_store_id: - if preferred_object_store_id not in object_store.object_store_ids_allowing_selection(): - return "Supplied object store id is not an allowed object store selection" - return "" +def validate_preferred_object_store_id( + trans, object_store: ObjectStore, preferred_object_store_id: Optional[str] +) -> str: + return object_store.validate_selected_object_store_id(trans.user, preferred_object_store_id) or "" diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index ee57aa2d1100..23c48a16423d 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -81,6 +81,19 @@ def list_secrets(self, key: str) -> List[str]: Note that only immediate subkeys are returned. """ + def delete_secret(self, key: str) -> None: + """ + Eliminate a secret from the target vault. + + Ideally the entry in the target source if removed, but by default the secret is + simply overwritten with the empty string as its value. + + :param key: The key to write to. Typically a hierarchical path such as `/galaxy/user/1/preferences/editor` + :param value: The value to write, such as 'vscode' + :return: + """ + self.write_secret(key, "") + class NullVault(Vault): def read_secret(self, key: str) -> Optional[str]: @@ -161,6 +174,11 @@ def write_secret(self, key: str, value: str) -> None: token = f.encrypt(value.encode("utf-8")) self._update_or_create(key=key, value=token.decode("utf-8")) + def delete_secret(self, key: str) -> None: + vault_entry = self.sa_session.query(model.Vault).filter_by(key=key).first() + self.sa_session.delete(vault_entry) + self.sa_session.flush() + def list_secrets(self, key: str) -> List[str]: raise NotImplementedError() @@ -299,3 +317,7 @@ def from_app(app) -> Vault: return VaultFactory.from_vault_type(app, vault_config.get("type", None), vault_config) log.warning("No vault configured. We recommend defining the vault_config_file setting in galaxy.yml") return NullVault() + + +def is_vault_configured(vault: Vault) -> bool: + return not isinstance(vault, NullVault) diff --git a/lib/galaxy/webapps/galaxy/api/object_store.py b/lib/galaxy/webapps/galaxy/api/object_store.py index 31f067dbfb76..26627de09441 100644 --- a/lib/galaxy/webapps/galaxy/api/object_store.py +++ b/lib/galaxy/webapps/galaxy/api/object_store.py @@ -6,6 +6,7 @@ from typing import List from fastapi import ( + Body, Path, Query, ) @@ -15,10 +16,17 @@ RequestParameterInvalidException, ) from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.object_store_instances import ( + CreateInstancePayload, + ModifyInstancePayload, + ObjectStoreInstancesManager, + UserConcreteObjectStoreModel, +) from galaxy.objectstore import ( BaseObjectStore, ConcreteObjectStoreModel, ) +from galaxy.objectstore.templates import ObjectStoreTemplateSummaries from . import ( depends, DependsOnTrans, @@ -27,12 +35,16 @@ log = logging.getLogger(__name__) -router = Router(tags=["object sstore"]) +router = Router(tags=["object_stores"]) ConcreteObjectStoreIdPathParam: str = Path( ..., title="Concrete Object Store ID", description="The concrete object store ID." ) +UserObjectStoreIdPathParam: str = Path( + ..., title="User Object Store ID", description="The model ID for a persisted UserObjectStore object." +) + SelectableQueryParam: bool = Query( default=False, title="Selectable", @@ -43,6 +55,7 @@ @router.cbv class FastAPIObjectStore: object_store: BaseObjectStore = depends(BaseObjectStore) + object_store_instance_manager: ObjectStoreInstancesManager = depends(ObjectStoreInstancesManager) @router.get( "/api/object_stores", @@ -59,7 +72,47 @@ def index( "The object store index query currently needs to be called with selectable=true" ) selectable_ids = self.object_store.object_store_ids_allowing_selection() - return [self._model_for(selectable_id) for selectable_id in selectable_ids] + instances = [self._model_for(selectable_id) for selectable_id in selectable_ids] + if trans.user: + user_object_stores = trans.user.object_stores + for user_object_store in user_object_stores: + instances.append(self.object_store_instance_manager._to_model(trans, user_object_store)) + return instances + + @router.post( + "/api/object_store_instances", + summary="Create a user-bound object store.", + operation_id="object_stores__create_instance", + ) + def create( + self, + trans: ProvidesUserContext = DependsOnTrans, + payload: CreateInstancePayload = Body(...), + ) -> UserConcreteObjectStoreModel: + return self.object_store_instance_manager.create_instance(trans, payload) + + @router.get( + "/api/object_store_instances", + summary="Get a list of persisted object store instances defined by the requesting user.", + operation_id="object_stores__instances_index", + ) + def instance_index( + self, + trans: ProvidesUserContext = DependsOnTrans, + ) -> List[UserConcreteObjectStoreModel]: + return self.object_store_instance_manager.index(trans) + + @router.get( + "/api/object_store_instances/{user_object_store_id}", + summary="Get a list of persisted object store instances defined by the requesting user.", + operation_id="object_stores__instances_get", + ) + def instances_show( + self, + trans: ProvidesUserContext = DependsOnTrans, + user_object_store_id: str = UserObjectStoreIdPathParam, + ) -> UserConcreteObjectStoreModel: + return self.object_store_instance_manager.show(trans, user_object_store_id) @router.get( "/api/object_stores/{object_store_id}", @@ -72,6 +125,31 @@ def show_info( ) -> ConcreteObjectStoreModel: return self._model_for(object_store_id) + @router.put( + "/api/object_store_instances/{user_object_store_id}", + summary="Update or upgrade user object store instance.", + operation_id="object_stores__instances_update", + ) + def update_instance( + self, + trans: ProvidesUserContext = DependsOnTrans, + user_object_store_id: str = UserObjectStoreIdPathParam, + payload: ModifyInstancePayload = Body(...), + ) -> UserConcreteObjectStoreModel: + return self.object_store_instance_manager.modify_instance(trans, user_object_store_id, payload) + + @router.get( + "/api/object_store_templates", + summary="Get a list of object store templates available to build user defined object stores from", + response_description="A list of the configured object store templates.", + operation_id="object_stores__templates_index", + ) + def index_templates( + self, + trans: ProvidesUserContext = DependsOnTrans, + ) -> ObjectStoreTemplateSummaries: + return self.object_store_instance_manager.summaries + def _model_for(self, object_store_id: str) -> ConcreteObjectStoreModel: concrete_object_store = self.object_store.get_concrete_store_by_object_store_id(object_store_id) if concrete_object_store is None: diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 2ff3ea8c2e1e..187125846594 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -222,6 +222,11 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): webapp.add_client_route("/user") webapp.add_client_route("/user/notifications{path:.*?}") webapp.add_client_route("/user/{form_id}") + webapp.add_client_route("/object_store_instances/create") + webapp.add_client_route("/object_store_instances/index") + webapp.add_client_route("/object_store_instances/{user_object_store_id}/edit") + webapp.add_client_route("/object_store_instances/{user_object_store_id}/upgrade") + webapp.add_client_route("/object_store_templates/{template_id}/new") webapp.add_client_route("/welcome/new") webapp.add_client_route("/visualizations") webapp.add_client_route("/visualizations/edit") diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 3bad7b7f034f..87b434eb8369 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -1168,9 +1168,13 @@ def get_usage_for(self, label: Optional[str]) -> Dict[str, Any]: def update_user(self, properties: Dict[str, Any]) -> Dict[str, Any]: update_response = self.update_user_raw(properties) - update_response.raise_for_status() + api_asserts.assert_status_code_is_ok(update_response) return update_response.json() + def set_user_preferred_object_store_id(self, store_id: Optional[str]) -> None: + user_properties = self.update_user({"preferred_object_store_id": store_id}) + assert user_properties["preferred_object_store_id"] == store_id + def update_user_raw(self, properties: Dict[str, Any]) -> Response: update_response = self.galaxy_interactor.put("users/current", properties, json=True) return update_response @@ -1462,6 +1466,36 @@ def wait_for_dataset( timeout=timeout, ) + def create_object_store_raw(self, payload: Dict[str, Any]) -> Response: + response = self._post( + "/api/object_store_instances", + payload, + json=True, + ) + return response + + def create_object_store(self, payload: Dict[str, Any]) -> Dict[str, Any]: + response = self.create_object_store_raw(payload) + api_asserts.assert_status_code_is_ok(response) + return response.json() + + def upgrade_object_store_raw(self, id: Union[str, int], payload: Dict[str, Any]) -> Response: + response = self._put( + f"/api/object_store_instances/{id}", + payload, + json=True, + ) + return response + + def upgrade_object_store(self, id: Union[str, int], payload: Dict[str, Any]) -> Dict[str, Any]: + response = self.upgrade_object_store_raw(id, payload) + api_asserts.assert_status_code_is_ok(response) + return response.json() + + # same implementation client side, slightly different types... + update_object_store_raw = upgrade_object_store_raw + update_object_store = upgrade_object_store + def selectable_object_stores(self) -> List[Dict[str, Any]]: selectable_object_stores_response = self._get("object_stores?selectable=true") selectable_object_stores_response.raise_for_status() diff --git a/lib/galaxy_test/driver/integration_util.py b/lib/galaxy_test/driver/integration_util.py index d5a159dc0324..15c604f4f4d1 100644 --- a/lib/galaxy_test/driver/integration_util.py +++ b/lib/galaxy_test/driver/integration_util.py @@ -222,14 +222,20 @@ class ConfiguresObjectStores: object_stores_parent: ClassVar[str] _test_driver: GalaxyTestDriver + @classmethod + def write_object_store_config_file(cls, filename: str, contents: str) -> str: + temp_directory = cls.object_stores_parent + config_path = os.path.join(temp_directory, filename) + with open(config_path, "w") as f: + f.write(contents) + return config_path + @classmethod def _configure_object_store(cls, template, config): temp_directory = cls._test_driver.mkdtemp() cls.object_stores_parent = temp_directory - config_path = os.path.join(temp_directory, "object_store_conf.xml") xml = template.safe_substitute({"temp_directory": temp_directory}) - with open(config_path, "w") as f: - f.write(xml) + config_path = cls.write_object_store_config_file("object_store_conf.xml", xml) config["object_store_config_file"] = config_path for path in re.findall(r'files_dir path="([^"]*)"', xml): assert path.startswith(temp_directory) @@ -238,6 +244,12 @@ def _configure_object_store(cls, template, config): safe_makedirs(path) setattr(cls, f"{dir_name}_path", path) + @classmethod + def _configure_object_store_template_catalog(cls, catalog, config): + template = catalog.replace("/data", cls.object_stores_parent) + template_config_path = cls.write_object_store_config_file("templates.yml", template) + config["object_store_templates_config_file"] = template_config_path + class ConfiguresDatabaseVault: @classmethod diff --git a/packages/objectstore/setup.cfg b/packages/objectstore/setup.cfg index 496b60c2ce75..0b47a88eaa10 100644 --- a/packages/objectstore/setup.cfg +++ b/packages/objectstore/setup.cfg @@ -36,6 +36,7 @@ install_requires = galaxy-util pydantic>=2,!=2.6.0,!=2.6.1 PyYAML + Jinja2 packages = find: python_requires = >=3.7 diff --git a/pyproject.toml b/pyproject.toml index 34faa14f63e6..3780f46ba3b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ h5grove = ">=1.2.1" h5py = "*" importlib-resources = { version = "*", python = "<3.9" } # for importlib.{abc.Traversable,resources.{files, Package}} isa-rwval = ">=0.10.10" +Jinja2 = "*" kombu = ">=5.3" # for importlib-metadata fix isal = "*" lagom = "*" diff --git a/test/integration/objectstore/test_per_user.py b/test/integration/objectstore/test_per_user.py new file mode 100644 index 000000000000..b252d8926d7a --- /dev/null +++ b/test/integration/objectstore/test_per_user.py @@ -0,0 +1,452 @@ +from typing import ( + Any, + Dict, + Tuple, +) + +from galaxy_test.base import api_asserts +from galaxy_test.driver import integration_util +from ._base import BaseObjectStoreIntegrationTestCase +from .test_selection_with_resource_parameters import DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE + +LIBRARY_2 = """ +- id: general_disk + name: General Disk + description: General Disk Bound to You + configuration: + type: disk + files_dir: '/data/general/{{ user.username }}' + badges: + - type: more_secure + - type: faster +- id: secure_disk + name: Secure Disk + description: Secure Disk Bound to You + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}' + badges: + - type: more_secure + - type: slower +""" + + +LIBRARY_WITH_SECRET = """ +- id: secure_disk + name: Secure Disk + description: Secure Disk Bound to You + secrets: + sec1: + help: This is my test secret. + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}/{{ secrets.sec1 }}/aftersec' + badges: + - type: more_secure + - type: slower +""" + + +# These should not be things that affect the path... it is more for like +# enabling new connection features, etc... +MULTI_VERSION_LIBRARY = """ +- id: general_disk + version: 0 + name: General Disk (ver 0) + description: General Disk Bound to You + variables: + var_1: + type: string + help: Variable 1. + configuration: + type: disk + files_dir: '/data/version1/{{ variables.var_1 }}' +- id: general_disk + version: 1 + name: General Disk (ver 1) + description: General Disk Bound to You + variables: + var_1: + type: string + help: Variable 1. + var_2: + type: string + help: Variable 2. + configuration: + type: disk + files_dir: '/data/version1/{{ variables.var_1 }}_{{ variables.var_2 }}' +""" + + +MULTI_VERSION_WITH_SECRETS_LIBRARY = """ +- id: secure_disk + version: 0 + name: Secure Disk + description: Secure Disk Bound to You + secrets: + sec1: + help: This is my test secret. + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}/{{ secrets.sec1 }}/aftersec' + badges: + - type: more_secure + - type: slower +- id: secure_disk + version: 1 + name: Secure Disk + description: Secure Disk Bound to You + secrets: + sec1: + help: This is my test secret. + sec2: + help: This is my test secret 2. + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}/{{ secrets.sec1 }}/{{ secrets.sec2 }}' + badges: + - type: more_secure + - type: slower +- id: secure_disk + version: 2 + name: Secure Disk + description: Secure Disk Bound to You + secrets: + sec2: + help: This is my test secret 2. + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}/newbar/{{ secrets.sec2 }}' + badges: + - type: more_secure + - type: slower +""" + + +class BaseUserObjectStoreIntegration(BaseObjectStoreIntegrationTestCase): + def _create_simple_payload(self) -> Dict[str, Any]: + body = { + "name": "My Cool Disk", + "template_id": "general_disk", + "template_version": 0, + "secrets": {}, + "variables": {}, + } + return body + + def _create_simple_object_store(self) -> str: + before_selectable_object_store_count = len(self.dataset_populator.selectable_object_stores()) + + body = self._create_simple_payload() + object_store_json = self.dataset_populator.create_object_store(body) + assert "name" in object_store_json + assert object_store_json["name"] == "My Cool Disk" + object_store_id = object_store_json["object_store_id"] + assert object_store_id.startswith("user_objects://") + + object_stores = self.dataset_populator.selectable_object_stores() + after_selectable_object_store_count = len(object_stores) + assert after_selectable_object_store_count == before_selectable_object_store_count + 1 + return object_store_id + + def _create_hda_get_storage_info(self, history_id: str): + hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") + self.dataset_populator.wait_for_history(history_id) + return self.dataset_populator.dataset_storage_info(hda1["id"]), hda1 + + def _run_tool_with_object_store_id_and_then_revert(self, history_id: str, object_store_id: str): + storage_info, hda1 = self._create_hda_get_storage_info(history_id) + assert storage_info["object_store_id"] == "default" + + self.dataset_populator.set_user_preferred_object_store_id(object_store_id) + + def _run_tool(tool_id, inputs, preferred_object_store_id=None): + response = self.dataset_populator.run_tool( + tool_id, + inputs, + history_id, + preferred_object_store_id=preferred_object_store_id, + ) + self.dataset_populator.wait_for_history(history_id) + return response + + hda1_input = {"src": "hda", "id": hda1["id"]} + response = _run_tool("multi_data_param", {"f1": hda1_input, "f2": hda1_input}) + storage_info, output = self._storage_info_for_job_output(response) + self.dataset_populator.set_user_preferred_object_store_id(None) + return storage_info, output + + def _storage_info_for_job_output(self, job_dict) -> Tuple[Dict[str, Any], Dict[str, Any]]: + outputs = job_dict["outputs"] # could be a list or dictionary depending on source + try: + output = outputs[0] + except KeyError: + output = list(outputs.values())[0] + storage_info = self.dataset_populator.dataset_storage_info(output["id"]) + return storage_info, output + + @classmethod + def _write_template_and_object_store_config(cls, config, catalog: str): + cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config) + cls._configure_object_store_template_catalog(catalog, config) + + def _get_dataset_filename(self, history_id: str, output: Dict[str, Any]) -> str: + details = self.dataset_populator.get_history_dataset_details(history_id, dataset_id=output["id"]) + assert "file_name" in details + file_name = details["file_name"] + assert file_name + return file_name + + +class TestPerUserObjectStoreIntegration(BaseUserObjectStoreIntegration): + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls._write_template_and_object_store_config(config, LIBRARY_2) + + def test_create_and_update(self): + body = self._create_simple_payload() + object_store_json = self.dataset_populator.create_object_store(body) + assert object_store_json["template_version"] == 0 + badges = object_store_json["badges"] + badge_types = set([b["type"] for b in badges]) + assert "user_defined" in badge_types + assert "restricted" in badge_types + assert "cloud" not in badge_types + assert "faster" in badge_types + assert "more_secure" in badge_types + assert "no_quota" in badge_types + persisted_object_store_id = object_store_json["id"] + + payload = { + "name": "my new name", + "description": "my new description", + } + response = self.dataset_populator.update_object_store(persisted_object_store_id, payload) + assert response["name"] == "my new name" + assert response["description"] == "my new description" + assert response["template_version"] == 0 + + def test_create_and_use_simple(self): + object_store_id = self._create_simple_object_store() + with self.dataset_populator.test_history() as history_id: + storage_info, hda1 = self._create_hda_get_storage_info(history_id) + assert storage_info["object_store_id"] == "default" + self.dataset_populator.set_user_preferred_object_store_id(object_store_id) + + def _run_tool(tool_id, inputs, preferred_object_store_id=None): + response = self.dataset_populator.run_tool( + tool_id, + inputs, + history_id, + preferred_object_store_id=preferred_object_store_id, + ) + self.dataset_populator.wait_for_history(history_id) + return response + + hda1_input = {"src": "hda", "id": hda1["id"]} + response = _run_tool("multi_data_param", {"f1": hda1_input, "f2": hda1_input}) + storage_info, output = self._storage_info_for_job_output(response) + assert storage_info["object_store_id"] == object_store_id + contents = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) + assert contents.startswith("1 2 3") + self.dataset_populator.set_user_preferred_object_store_id(None) + + def test_create_unknown_id(self): + body = { + "template_id": "general_disk_2", + "template_version": 0, + "name": "My Unknown Disk", + "secrets": {}, + "variables": {}, + } + response = self.dataset_populator.create_object_store_raw(body) + api_asserts.assert_status_code_is(response, 404) + + def test_create_invalid_version(self): + body = { + "template_id": "general_disk", + "template_version": "0.0.0", + "name": "My Unknown Disk", + "secrets": {}, + "variables": {}, + } + response = self.dataset_populator.create_object_store_raw(body) + api_asserts.assert_status_code_is(response, 400) + + +class TestPerUserObjectStoreWithSecretsIntegration( + BaseUserObjectStoreIntegration, integration_util.ConfiguresDatabaseVault +): + # so we can see paths in the API... + require_admin_user = True + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls._configure_database_vault(config) + cls._write_template_and_object_store_config(config, LIBRARY_WITH_SECRET) + + def test_creation_with_secrets(self): + body = { + "name": "My Cool Disk", + "template_id": "secure_disk", + "template_version": 0, + "secrets": { + "sec1": "foobar", + }, + "variables": {}, + } + object_store_json = self.dataset_populator.create_object_store(body) + object_store_id = object_store_json["object_store_id"] + persisted_object_store_id = object_store_json["id"] + + with self.dataset_populator.test_history() as history_id: + _, output = self._run_tool_with_object_store_id_and_then_revert(history_id, object_store_id) + file_name = self._get_dataset_filename(history_id, output) + assert "foobar" in file_name + + update_payload = { + "secret_name": "sec1", + "secret_value": "newbar", + } + self.dataset_populator.update_object_store(persisted_object_store_id, update_payload) + + _, output = self._run_tool_with_object_store_id_and_then_revert(history_id, object_store_id) + file_name = self._get_dataset_filename(history_id, output) + assert "foobar" not in file_name + assert "newbar" in file_name + + +class TestPerUserObjectStoreWithExtendedMetadataIntegration(BaseUserObjectStoreIntegration): + """This requires serializing the object store... + + ...so there is a lot of complexity behind the scenes tested here.""" + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls._write_template_and_object_store_config(config, LIBRARY_2) + config["metadata_strategy"] = "extended" + # config["tool_evaluation_strategy"] = "remote" + config["retry_metadata_internally"] = False + + def test_create_and_use(self): + object_store_id = self._create_simple_object_store() + with self.dataset_populator.test_history() as history_id: + storage_info, output = self._run_tool_with_object_store_id_and_then_revert(history_id, object_store_id) + assert storage_info["object_store_id"] == object_store_id + contents = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) + assert contents.startswith("1 2 3") + self.dataset_populator.set_user_preferred_object_store_id(None) + + +class TestPerUserObjectStoreUpgradesIntegration(BaseUserObjectStoreIntegration): + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls._write_template_and_object_store_config(config, MULTI_VERSION_LIBRARY) + + def test_create_and_upgrade(self): + body = { + "name": "My Upgradable Disk", + "template_id": "general_disk", + "template_version": 0, + "secrets": {}, + "variables": { + "var_1": "moo_cow", + }, + } + object_store_json = self.dataset_populator.create_object_store(body) + assert "name" in object_store_json + assert object_store_json["name"] == "My Upgradable Disk" + assert object_store_json["template_version"] == 0 + + id = object_store_json["id"] + object_store_id = object_store_json["object_store_id"] + assert object_store_id.startswith("user_objects://") + + object_stores = self.dataset_populator.selectable_object_stores() + assert len(object_stores) == 1 + user_object_store = object_stores[0] + assert user_object_store["name"] == "My Upgradable Disk" + + body = { + "template_version": 1, + "secrets": {}, + "variables": { + "var_1": "moo", + "var_2": "cow", + }, + } + object_store_json = self.dataset_populator.upgrade_object_store(id, body) + assert "name" in object_store_json + assert object_store_json["name"] == "My Upgradable Disk" + new_object_store_id = object_store_json["object_store_id"] + assert new_object_store_id == object_store_id + assert object_store_json["id"] == id + assert object_store_json["template_version"] == 1 + + +class TestPerUserObjectStoreUpgradesWithSecretsIntegration( + BaseUserObjectStoreIntegration, integration_util.ConfiguresDatabaseVault +): + require_admin_user = True + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls._configure_database_vault(config) + cls._write_template_and_object_store_config(config, MULTI_VERSION_WITH_SECRETS_LIBRARY) + + def test_create_and_upgrade(self): + body = { + "name": "My Upgradable Disk", + "template_id": "secure_disk", + "template_version": 0, + "secrets": { + "sec1": "moocow", + }, + "variables": {}, + } + object_store_json = self.dataset_populator.create_object_store(body) + assert "name" in object_store_json + assert object_store_json["name"] == "My Upgradable Disk" + assert object_store_json["template_version"] == 0 + id = object_store_json["id"] + object_store_id = object_store_json["object_store_id"] + + secrets = object_store_json["secrets"] + assert "sec1" in secrets + assert "sec2" not in secrets + + with self.dataset_populator.test_history() as history_id: + _, output = self._run_tool_with_object_store_id_and_then_revert(history_id, object_store_id) + file_name = self._get_dataset_filename(history_id, output) + assert "moocow/aftersec" in file_name + assert "moocow/aftersec2" not in file_name + + body = { + "template_version": 1, + "secrets": { + "sec1": "moocow", + "sec2": "aftersec2", + }, + "variables": {}, + } + object_store_json = self.dataset_populator.upgrade_object_store(id, body) + secrets = object_store_json["secrets"] + assert object_store_json["template_version"] == 1 + assert "sec1" in secrets + assert "sec2" in secrets + + _, output = self._run_tool_with_object_store_id_and_then_revert(history_id, object_store_id) + file_name = self._get_dataset_filename(history_id, output) + + assert "moocow/aftersec2" in file_name + + body = { + "template_version": 2, + "secrets": {}, + "variables": {}, + } + object_store_json = self.dataset_populator.upgrade_object_store(id, body) + secrets = object_store_json["secrets"] + assert object_store_json["template_version"] == 2 + assert "sec1" not in secrets + assert "sec2" in secrets diff --git a/test/integration/objectstore/test_selection_with_user_preferred_object_store.py b/test/integration/objectstore/test_selection_with_user_preferred_object_store.py index 13cc0b912679..494bf093be72 100644 --- a/test/integration/objectstore/test_selection_with_user_preferred_object_store.py +++ b/test/integration/objectstore/test_selection_with_user_preferred_object_store.py @@ -470,9 +470,8 @@ def _storage_info_for_job_output(self, job_dict) -> Dict[str, Any]: def _storage_info(self, hda): return self.dataset_populator.dataset_storage_info(hda["id"]) - def _set_user_preferred_object_store_id(self, store_id: Optional[str]): - user_properties = self.dataset_populator.update_user({"preferred_object_store_id": store_id}) - assert user_properties["preferred_object_store_id"] == store_id + def _set_user_preferred_object_store_id(self, store_id: Optional[str]) -> None: + self.dataset_populator.set_user_preferred_object_store_id(store_id) def _reset_user_preferred_object_store_id(self): self._set_user_preferred_object_store_id(None) diff --git a/test/unit/objectstore/test_from_configuration_object.py b/test/unit/objectstore/test_from_configuration_object.py new file mode 100644 index 000000000000..e1e9dac6edee --- /dev/null +++ b/test/unit/objectstore/test_from_configuration_object.py @@ -0,0 +1,45 @@ +from galaxy.objectstore import ( + concrete_object_store, + UserObjectStoresAppConfig, +) +from galaxy.objectstore.templates.models import DiskObjectStoreConfiguration +from .test_objectstore import MockDataset + + +def test_disk(tmpdir): + files_dir = tmpdir / "moo" + files_dir.mkdir() + configuration = DiskObjectStoreConfiguration( + type="disk", + files_dir=str(files_dir), + ) + _app_config = app_config(tmpdir) + object_store = concrete_object_store(configuration, _app_config) + + absent_dataset = MockDataset(1) + assert not object_store.exists(absent_dataset) + + # Write empty dataset 2 in second backend, ensure it is empty and + # exists. + empty_dataset = MockDataset(2) + object_store.create(empty_dataset) + object_store.exists(empty_dataset) + assert object_store.size(empty_dataset) == 0 + + example_dataset = MockDataset(3) + temp_file = tmpdir / "example.txt" + temp_file.write_text("moo cow", "utf-8") + object_store.create(example_dataset) + object_store.update_from_file(example_dataset, file_name=str(temp_file)) + assert object_store.size(example_dataset) == 7 + + +def app_config(tmpdir) -> UserObjectStoresAppConfig: + app_config = UserObjectStoresAppConfig( + jobs_directory=str(tmpdir / "jobs"), + new_file_path=str(tmpdir / "new_files"), + umask=0o077, + object_store_cache_path=str(tmpdir / "cache"), + object_store_cache_size=1, + ) + return app_config diff --git a/test/unit/objectstore/test_serializing_user_object_stores.py b/test/unit/objectstore/test_serializing_user_object_stores.py new file mode 100644 index 000000000000..abbf5d50ba58 --- /dev/null +++ b/test/unit/objectstore/test_serializing_user_object_stores.py @@ -0,0 +1,64 @@ +from galaxy.objectstore import ( + BaseUserObjectStoreResolver, + build_object_store_from_config, + DistributedObjectStore, + serialize_static_object_store_config, +) +from galaxy.objectstore.templates.models import ( + DiskObjectStoreConfiguration, + ObjectStoreConfiguration, +) +from galaxy.objectstore.unittest_utils import Config +from .test_from_configuration_object import app_config +from .test_objectstore import MockDataset + +DISTRIBUTED_TEST_CONFIG_YAML = """ +type: distributed +backends: + - id: files1 + type: disk + weight: 1 + files_dir: "${temp_directory}/files1" + extra_dirs: + - type: temp + path: "${temp_directory}/tmp1" + - type: job_work + path: "${temp_directory}/job_working_directory1" +""" +TEST_URI = "user_objects://1" + + +class MockUserObjectStoreResolver(BaseUserObjectStoreResolver): + def __init__(self, tmpdir): + test_dir = tmpdir / "files" + test_dir.mkdir() + self.test_dir = test_dir + self._app_config = app_config(tmpdir) + + def resolve_object_store_uri_config(self, uri: str) -> ObjectStoreConfiguration: + assert uri == TEST_URI + files_dir = self.test_dir / "moo" + files_dir.mkdir(exist_ok=True) + configuration = DiskObjectStoreConfiguration( + type="disk", + files_dir=str(files_dir), + ) + return configuration + + +def test_serialize_and_repopulate(tmp_path): + resolver = MockUserObjectStoreResolver(tmp_path.resolve()) + with Config(DISTRIBUTED_TEST_CONFIG_YAML, user_object_store_resolver=resolver) as (directory, object_store): + dataset = MockDataset(id=1) + dataset.object_store_id = TEST_URI + object_store.create(dataset) + assert object_store.exists(dataset) + + object_store_uris = set([TEST_URI]) + as_dict = serialize_static_object_store_config(object_store, object_store_uris) + rehydrated_object_store = build_object_store_from_config(None, config_dict=as_dict) + + assert isinstance(rehydrated_object_store, DistributedObjectStore) + assert TEST_URI in rehydrated_object_store.backends + + assert rehydrated_object_store.exists(dataset) diff --git a/test/unit/objectstore/test_template_manager.py b/test/unit/objectstore/test_template_manager.py new file mode 100644 index 000000000000..18f2094192c9 --- /dev/null +++ b/test/unit/objectstore/test_template_manager.py @@ -0,0 +1,58 @@ +from galaxy.objectstore.templates import ConfiguredObjectStoreTemplates +from .test_template_models import ( + LIBRARY_2, + LIBRARY_AZURE_CONTAINER, +) + + +class MockConfig: + def __init__(self, config_path): + self.object_store_templates = None + self.object_store_templates_config_file = config_path + + +def test_manager(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_2, "utf-8") + config = MockConfig(config_path) + templates = ConfiguredObjectStoreTemplates.from_app_config(config) + summaries = templates.summaries + assert summaries + assert len(summaries.root) == 2 + + +def test_manager_throws_exception_if_vault_is_required_but_configured(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_AZURE_CONTAINER, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredObjectStoreTemplates.from_app_config(config, vault_configured=False) + except Exception as e: + exc = e + assert exc, "catalog creation should result in an exception" + assert "vault must be configured" in str(exc) + + +def test_manager_with_secrets_is_fine_if_vault_is_required_and_configured(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_AZURE_CONTAINER, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredObjectStoreTemplates.from_app_config(config, vault_configured=True) + except Exception as e: + exc = e + assert exc is None + + +def test_manager_does_not_throw_exception_if_vault_is_not_required(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_2, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredObjectStoreTemplates.from_app_config(config, vault_configured=False) + except Exception as e: + exc = e + assert exc is None diff --git a/test/unit/objectstore/test_template_models.py b/test/unit/objectstore/test_template_models.py new file mode 100644 index 000000000000..505f9b3e8d99 --- /dev/null +++ b/test/unit/objectstore/test_template_models.py @@ -0,0 +1,211 @@ +from yaml import safe_load + +from galaxy.objectstore.templates.manager import raw_config_to_catalog +from galaxy.objectstore.templates.models import ( + AzureObjectStoreConfiguration, + DiskObjectStoreConfiguration, + GenericS3ObjectStoreConfiguration, + ObjectStoreTemplateCatalog, + S3ObjectStoreConfiguration, + template_to_configuration, +) + +LIBRARY_1 = """ +- id: amazon_bucket + name: Amazon Bucket + description: An Amazon S3 Bucket + variables: + use_reduced_redundancy: + type: boolean + help: Reduce redundancy and save money. + secrets: + access_key: + help: AWS access key to use when connecting to AWS resources. + secret_key: + help: AWS secret key to use when connecting to AWS resources. + bucket_name: + help: Name of bucket to use when connecting to AWS resources. + configuration: + type: s3 + auth: + access_key: '{{ secrets.access_key}}' + secret_key: '{{ secrets.secret_key}}' + bucket: + name: '{{ secrets.bucket_name}}' + use_reduced_redundancy: '{{ variables.use_reduced_redundancy}}' + badges: + - type: less_stable + - type: slower + - type: not_backed_up +""" + + +def test_parsing_simple_s3(): + template_library = _parse_template_library(LIBRARY_1) + assert len(template_library.root) == 1 + s3_template = template_library.root[0] + assert s3_template.description == "An Amazon S3 Bucket" + configuration_obj = template_to_configuration( + s3_template, + {"use_reduced_redundancy": False}, + {"access_key": "sec1", "secret_key": "sec2", "bucket_name": "sec3"}, + user_details={}, + ) + badges = s3_template.configuration.badges + assert badges + assert len(badges) == 3 + + # expanded configuration should validate with template expansions... + assert isinstance(configuration_obj, S3ObjectStoreConfiguration) + configuration = configuration_obj.model_dump() + + assert configuration["type"] == "s3" + assert configuration["auth"]["access_key"] == "sec1" + assert configuration["auth"]["secret_key"] == "sec2" + assert configuration["bucket"]["name"] == "sec3" + assert configuration["bucket"]["use_reduced_redundancy"] is False + assert len(configuration["badges"]) == 3 + + +LIBRARY_GENERIC_S3 = """ +- id: minio_bucket + name: MinIO Bucket + description: A MinIO bucket connected using a generic S3 object store. + variables: + use_reduced_redundancy: + type: boolean + help: Reduce redundancy and save money. + secrets: + access_key: + help: AWS access key to use when connecting to AWS resources. + secret_key: + help: AWS secret key to use when connecting to AWS resources. + bucket_name: + help: Name of bucket to use when connecting to AWS resources. + configuration: + type: generic_s3 + auth: + access_key: '{{ secrets.access_key}}' + secret_key: '{{ secrets.secret_key}}' + bucket: + name: '{{ secrets.bucket_name}}' + use_reduced_redundancy: '{{ variables.use_reduced_redundancy}}' + connection: + host: minio.galaxyproject.org + port: 5679 + badges: + - type: less_stable + - type: slower + - type: not_backed_up +""" + + +def test_parsing_generic_s3(): + template_library = _parse_template_library(LIBRARY_GENERIC_S3) + assert len(template_library.root) == 1 + s3_template = template_library.root[0] + assert s3_template.description == "A MinIO bucket connected using a generic S3 object store." + configuration_obj = template_to_configuration( + s3_template, + {"use_reduced_redundancy": False}, + {"access_key": "sec1", "secret_key": "sec2", "bucket_name": "sec3"}, + user_details={}, + ) + badges = s3_template.configuration.badges + assert badges + assert len(badges) == 3 + + # expanded configuration should validate with template expansions... + assert isinstance(configuration_obj, GenericS3ObjectStoreConfiguration) + configuration = configuration_obj.model_dump() + + assert configuration["type"] == "generic_s3" + assert configuration["auth"]["access_key"] == "sec1" + assert configuration["auth"]["secret_key"] == "sec2" + assert configuration["bucket"]["name"] == "sec3" + assert configuration["bucket"]["use_reduced_redundancy"] is False + assert configuration["connection"]["host"] == "minio.galaxyproject.org" + assert configuration["connection"]["port"] == 5679 + assert configuration["connection"]["conn_path"] == "" + assert configuration["connection"]["is_secure"] is True + assert len(configuration["badges"]) == 3 + + +LIBRARY_2 = """ +- id: general_disk + name: General Disk + description: General Disk Bound to You + configuration: + type: disk + files_dir: '/data/general/{{ user.username }}' +- id: secure_disk + name: Secure Disk + description: Secure Disk Bound to You + configuration: + type: disk + files_dir: '/data/secure/{{ user.username }}' +""" + + +def test_parsing_multiple_posix(): + template_library = _parse_template_library(LIBRARY_2) + assert len(template_library.root) == 2 + general_template = template_library.root[0] + secure_template = template_library.root[1] + + assert general_template.version == 0 + assert secure_template.version == 0 + assert secure_template.hidden is False + + general_configuration = template_to_configuration(general_template, {}, {}, user_details={"username": "jane"}) + assert isinstance(general_configuration, DiskObjectStoreConfiguration) + assert general_configuration.files_dir == "/data/general/jane" + + secure_configuration = template_to_configuration(secure_template, {}, {}, user_details={"username": "jane"}) + assert isinstance(secure_configuration, DiskObjectStoreConfiguration) + assert secure_configuration.files_dir == "/data/secure/jane" + + +LIBRARY_AZURE_CONTAINER = """ +- id: amazon_bucket + name: Azure Container + description: An Azure Container + variables: + account_name: + type: string + help: Azure account name to use when connecting to Azure resources. + secrets: + account_key: + help: Azure account key to use when connecting to Azure resources. + container_name: + help: Name of container to use when connecting to Azure cloud resources. + configuration: + type: azure_blob + auth: + account_name: '{{ variables.account_name}}' + account_key: '{{ secrets.account_key}}' + container: + name: '{{ secrets.container_name}}' +""" + + +def test_parsing_azure(): + template_library = _parse_template_library(LIBRARY_AZURE_CONTAINER) + assert len(template_library.root) == 1 + azure_template = template_library.root[0] + assert azure_template.description == "An Azure Container" + configuration_obj = template_to_configuration( + azure_template, + {"account_name": "galaxyproject"}, + {"account_key": "sec1", "container_name": "sec2"}, + user_details={}, + ) + assert isinstance(configuration_obj, AzureObjectStoreConfiguration) + assert configuration_obj.auth.account_name == "galaxyproject" + assert configuration_obj.auth.account_key == "sec1" + assert configuration_obj.container.name == "sec2" + + +def _parse_template_library(contents: str) -> ObjectStoreTemplateCatalog: + raw_contents = safe_load(contents) + return raw_config_to_catalog(raw_contents)