Allow oauth2 file sources with Dropbox initial implementation.

client/src/components/ConfigTemplates/ConfigurationTestSummary.vue

@@ -15,6 +15,9 @@ defineProps<Props>();
 <template>
     <BListGroup v-if="testResults">
         <ConfigurationTestItem :status="testResults?.template_definition" />
+        <ConfigurationTestItem
+            v-if="testResults?.oauth2_access_token_generation != null"
+            :status="testResults?.oauth2_access_token_generation" />
         <ConfigurationTestItem :status="testResults?.template_settings" />
         <ConfigurationTestItem :status="testResults?.connection" />
     </BListGroup>

client/src/components/ConfigTemplates/useConfigurationTesting.ts

@@ -43,6 +43,7 @@ export function useConfigurationTesting() {
 export function useConfigurationTemplateCreation<T extends TemplateSummary, R>(
     what: string,
     template: Ref<T>,
+    uuid: Ref<string | undefined>,
     test: (payload: CreateInstancePayload) => Promise<{ data: PluginStatus }>,
     create: (payload: CreateInstancePayload) => Promise<{ data: R }>,
     onCreate: (result: R) => unknown
@@ -52,6 +53,9 @@ export function useConfigurationTemplateCreation<T extends TemplateSummary, R>(
 
     async function onSubmit(formData: any) {
         const payload = createFormDataToPayload(template.value, formData);
+        if (uuid.value) {
+            payload.uuid = uuid.value;
+        }
         let pluginStatus;
         try {
             testRunning.value = true;

client/src/components/FileSources/FileSourceTypeSpan.vue

@@ -4,11 +4,13 @@ import { computed } from "vue";
 import type { FileSourceTypes } from "@/api/fileSources";
 
 const MESSAGES = {
-    posix: "This is a simple path based storage location that assumes the all the relevant paths are already mounted on the Galaxy server and target worker nodes.",
+    posix: "This is a simple path based file source that assumes the all the relevant paths are already mounted on the Galaxy server and target worker nodes.",
     s3fs: "This is an remote file source plugin based on the Amazon Simple Storage Service (S3) interface. The AWS interface has become an industry standard and many storage vendors support it and use it to expose 'object' based storage.",
     azure: "This is an remote file source plugin based on the Azure service.",
     onedata: "This is an remote file source plugin based on the Onedata service.",
     ftp: "This is an remote file source plugin based on the FTP protocol.",
+    dropbox: "This is a file source plugin that connects with the commercial Dropbox service.",
+    googledrive: "This is a file source plugin that connects with the commercial Google Drive service.",
 };
 
 interface Props {

client/src/components/FileSources/Instances/CreateForm.vue

@@ -8,6 +8,7 @@ import { create, test } from "./services";
 
 interface CreateFormProps {
     template: FileSourceTemplateSummary;
+    uuid?: string;
 }
 const props = defineProps<CreateFormProps>();
 const title = "Create a new file source for your data";
@@ -20,6 +21,7 @@ const { ActionSummary, error, inputs, InstanceForm, onSubmit, submitTitle, loadi
     useConfigurationTemplateCreation(
         "file source",
         toRef(props, "template"),
+        toRef(props, "uuid"),
         test,
         create,
         (fileSource: UserFileSourceModel) => emit("created", fileSource)

client/src/components/FileSources/Instances/CreateInstance.vue

@@ -1,34 +1,63 @@
 <script setup lang="ts">
-import { computed } from "vue";
+import { computed, watch } from "vue";
 
 import type { UserFileSourceModel } from "@/api/fileSources";
 import { useFileSourceTemplatesStore } from "@/stores/fileSourceTemplatesStore";
 
 import { useInstanceRouting } from "./routing";
+import { getOAuth2Info } from "./services";
 
 import CreateForm from "@/components/FileSources/Instances/CreateForm.vue";
 import LoadingSpan from "@/components/LoadingSpan.vue";
 
 interface Props {
     templateId: string;
+    uuid?: string;
 }
+
+const OAUTH2_TYPES = ["dropbox", "googledrive"];
+
 const fileSourceTemplatesStore = useFileSourceTemplatesStore();
 fileSourceTemplatesStore.fetchTemplates();
 
 const { goToIndex } = useInstanceRouting();
 
 const props = defineProps<Props>();
 const template = computed(() => fileSourceTemplatesStore.getLatestTemplate(props.templateId));
+const requiresOAuth2AuthorizeRedirect = computed(() => {
+    const templateValue = template.value;
+    return props.uuid == undefined && templateValue && OAUTH2_TYPES.indexOf(templateValue.type) >= 0;
+});
 
 async function onCreated(objectStore: UserFileSourceModel) {
     const message = `Created file source ${objectStore.name}`;
     goToIndex({ message });
 }
+
+watch(
+    requiresOAuth2AuthorizeRedirect,
+    async (requiresAuth) => {
+        const templateValue = template.value;
+        if (templateValue && requiresAuth) {
+            const { data } = await getOAuth2Info({
+                template_id: templateValue.id,
+                template_version: templateValue.version || 0,
+            });
+            window.location.href = data.authorize_url;
+        } else {
+            console.log("skipping this...");
+        }
+    },
+    { immediate: true }
+);
 </script>
 
 <template>
     <div>
         <LoadingSpan v-if="!template" message="Loading file source templates" />
-        <CreateForm v-else :template="template" @created="onCreated"></CreateForm>
+        <LoadingSpan
+            v-else-if="requiresOAuth2AuthorizeRedirect"
+            message="Fetching redirect information, you will need to authorize Galaxy to have access to this resource remotely" />
+        <CreateForm v-else :uuid="uuid" :template="template" @created="onCreated"></CreateForm>
     </div>
 </template>

client/src/components/FileSources/Instances/services.ts

@@ -9,6 +9,10 @@ export const testInstance = fetcher
     .create();
 export const update = fetcher.path("/api/file_source_instances/{user_file_source_id}").method("put").create();
 export const testUpdate = fetcher.path("/api/file_source_instances/{user_file_source_id}/test").method("post").create();
+export const getOAuth2Info = fetcher
+    .path("/api/file_source_templates/{template_id}/{template_version}/oauth2")
+    .method("get")
+    .create();
 
 export async function hide(instance: UserFileSourceModel) {
     const payload = { hidden: true };

client/src/components/ObjectStore/Instances/CreateForm.vue

@@ -9,6 +9,7 @@ import { create, test } from "./services";
 
 interface CreateFormProps {
     template: ObjectStoreTemplateSummary;
+    uuid?: string;
 }
 const props = defineProps<CreateFormProps>();
 const title = "Create a new storage location for your data";
@@ -21,6 +22,7 @@ const { ActionSummary, error, inputs, InstanceForm, onSubmit, submitTitle, loadi
     useConfigurationTemplateCreation(
         "storage location",
         toRef(props, "template"),
+        toRef(props, "uuid"),
         test,
         create,
         (instance: UserConcreteObjectStore) => emit("created", instance)

client/src/entry/analysis/router.js

@@ -414,7 +414,10 @@ export function getRouter(Galaxy) {
                     {
                         path: "file_source_templates/:templateId/new",
                         component: CreateFileSourceInstance,
-                        props: true,
+                        props: (route) => ({
+                            templateId: route.params.templateId,
+                            uuid: route.query.uuid,
+                        }),
                     },
                     {
                         path: "pages/create",

doc/source/admin/data.md

@@ -350,6 +350,19 @@ configuration).
 
 ![](file_source_azure_configuration.png)
 
+#### ``dropbox``
+
+The syntax for the ``configuration`` section of ``dropbox`` templates looks like this.
+
+![](file_source_dropbox_configuration_template.png)
+
+At runtime, after the ``configuration`` template is expanded, the resulting dictionary
+passed to Galaxy's file source plugin infrastructure looks like this and should match a subset
+of what you'd be able to add directly to ``file_sources_conf.yml`` (Galaxy's global file source
+configuration).
+
+![](file_source_dropbox_configuration.png)
+
 ### YAML Syntax
 
 ![galaxy.files.templates.models](file_source_templates.png)
@@ -398,6 +411,74 @@ and you are comfortable with it storing your user's secrets.
 :language: yaml
 ```
 
+### Production OAuth 2.0 File Source Templates
+
+Unlike the examples in the previous section. These examples will require a bit of
+configuration on the part of the admin. This is to obtain client credentials from the
+external service and register an OAuth 2.0 redirection callback with the remote service.
+
+#### Dropbox
+
+Once you have OAuth 2.0 client credentials from Dropbox (called ``oauth2_client_id`` and
+``oauth2_client_secret`` here), the following configuration can be used configure your Galaxy
+instance to enable Dropbox.
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_dropbox.yml
+:language: yaml
+```
+
+To use this template - you'll need to make your credentials available to Galaxy's
+web and job handler processes using the environment variables ``GALAXY_DROPBOX_APP_CLIENT_ID``
+and ``GALAXY_DROPBOX_APP_CLIENT_SECRET``. Your jobs themselves do not require
+these secrets to be set and will not be given the secrets.
+
+If you'd like to configure these secrets explicit - you can configure them explicitly
+in the configuration. If your configuration file is managed by Ansible, these secrets
+could potentially be populated from your Ansible vault.
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/dropbox_client_secrets_explicit.yml
+:language: yaml
+```
+
+To obtain the OAuth 2.0 credentials from Dropbox, you'll need to navigate to your
+[Dropbox Apps](https://www.dropbox.com/developers/apps) and create a new app 
+for your Galaxy instance with the "Create app" button.
+
+![Screenshot of Dropbox](dropbox_create_button.png)
+
+The only option available is "Scoped access" and this works fine for typical
+Galaxy use cases. You will however want to click "Full Dropbox" to request
+full access to your user's account. You will also need to give your "App" a name
+here, this should likely be something related to your Galaxy instances name.
+
+![Screenshot of Dropbox](dropbox_create_app_options_1.png)
+
+After your app is created, you'll be presented with a management screen for it.
+The first thing you'll want to do is navigate to the permissions tab and enable
+permissions to read and write to files and directories so the file source plugin
+works properly:
+
+![Screenshot of Dropbox](dropbox_scopes.png)
+
+Next, navigate back to the "Settings" tab. You'll need to register a callback
+for your Galaxy instance (it will need HTTPS enabled). This should be the URL
+to your Galaxy instance with ``oauth2_callback`` appended to it.
+
+![Screenshot of Dropbox](dropbox_callback.png)
+
+Finally you'll be able to find the ``oauth2_client_id`` and ``oauth2_client_secret``
+to configured your Galaxy with on this settings page.
+
+![Screenshot of Dropbox](dropbox_client_creds.png)
+
+Until you have 50 users, your App will be considered a "development" application.
+The upshot of this is that your user's will get a scary message during authorization
+but there seems to be no way around this. 50 users would definitely be considered
+a production Galaxy instance but Dropbox operates on a different scale.
+
+For more information on what Dropbox considers a "development" app versus a "production"
+app - checkout the [Dropbox documentation](https://www.dropbox.com/developers/reference/developer-guide#production-approval).
+
 ## Playing Nicer with Ansible
 
 Many large instances of Galaxy are configured with Ansible and much of the existing administrator
@@ -604,3 +685,70 @@ variable is not defined at runtime.
 ```{literalinclude} ../../../lib/galaxy/files/templates/examples/admin_secrets_with_defaults.yml
 :language: yaml
 ```
+
+## OAuth 2.0 Enabled Configurations
+
+[OAuth 2.0](https://oauth.net/2/) has become an industry standard for allowing users
+of various services (e.g. Dropbox or Google Drive) to authorize other services (e.g. Galaxy)
+fine grained access to the services. There is a bit of a dance the services need to do
+but the result can be a fairly nice end-user experience. The framework for configuring
+user defined data access templates can support OAuth 2.0.
+
+Galaxy keeps track of which plugin ``type``s (currently only file source types) require
+OAuth2 to work properly and will take care of authorization redirection, saving refresh tokens,
+etc.. implicitly. One such ``type`` is ``dropbox``. Here is the production Dropbox
+template distributed with Galaxy.
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_dropbox.yml
+:language: yaml
+```
+
+OAuth2 enabled plugin types include template definitions that include ``oauth2_client_id``
+and ``oauth2_client_secret`` in the configuration (as shown in the following specification
+and in the above examples).
+
+![](file_source_dropbox_configuration_template.png)
+
+The above example defines these secrets using environment variables but they can stored in
+Galaxy's Vault explicitly by the admin or written right to the configuration files as shown
+in the next two examples:
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/dropbox_client_secrets_in_vault.yml
+:language: yaml
+```
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/dropbox_client_secrets_explicit.yml
+:language: yaml
+```
+
+Looking at the configuration objects that get generated at runtime
+from these templates though - ``oauth2_client_id`` and ``oauth2_client_secret`` no longer
+appear and instead have been replaced with a ``oauth2_access_token`` parameter.
+Galaxy will take care of stripping out the client (e.g. Galaxy server) information and
+replacing it with short-term access tokens generated for the user's resources.
+
+![](file_source_dropbox_configuration.png)
+
+Normally, a UUID is created for each user configured instance object and this is used
+to store the template's explicitly listed secrets in Galaxy's Vault. For OAuth 2.0
+plugin types - before user's are even prompted for configuration metadata they are redirected
+to the remote service and prompted to authorize Galaxy to act on their behalf when using
+the remote service. If they authorize this, the remote service will send an [`authorization
+code`](https://oauth.net/2/grant-types/authorization-code/) to ``https://<galaxy_url>/oauth2_callback`` along with state information
+to recover which instance is being configured. At this point, Galaxy will fetch a [`refresh token`](https://oauth.net/2/refresh-tokens/) from the remote resource using the
+supplied authorization code. The refresh token is stored in the Vault in key associated with
+the UUID of the object that will be created when the user finishes the creation process.
+Specifically it is stored at 
+
+```
+/galaxy/user/<user_id>/file_source_config/<file_source_instance_uuid>/_oauth2_refresh_token
+```
+
+Here is the prefix at the end of ``_`` is indicating that Galaxy is managing this instead of
+it being listed explicitly in a ``secrets`` section of the template configuration like the
+explicit Vault secrets discussed in this document.
+
+Galaxy knows how to fetch an [`access token`](https://oauth.net/2/access-tokens/) from this
+refresh token that is actually used to interact with the remote resource. This is the property
+``oauth2_access_token`` that is injected into the configuration object shown above and passed
+along to the actual object store or file source plugin implementation.

doc/source/admin/gen_diagrams.py

@@ -8,6 +8,8 @@
 from galaxy.files.templates.models import (
     AzureFileSourceConfiguration,
     AzureFileSourceTemplateConfiguration,
+    DropboxFileSourceConfiguration,
+    DropboxFileSourceTemplateConfiguration,
     FileSourceTemplate,
     FtpFileSourceConfiguration,
     FtpFileSourceTemplateConfiguration,
@@ -47,6 +49,8 @@
     FileSourceTemplate: "file_source_templates",
     AzureFileSourceTemplateConfiguration: "file_source_azure_configuration_template",
     AzureFileSourceConfiguration: "file_source_azure_configuration",
+    DropboxFileSourceTemplateConfiguration: "file_source_dropbox_configuration_template",
+    DropboxFileSourceConfiguration: "file_source_dropbox_configuration",
     PosixFileSourceTemplateConfiguration: "file_source_posix_configuration_template",
     PosixFileSourceConfiguration: "file_source_posix_configuration",
     S3FSFileSourceTemplateConfiguration: "file_source_s3fs_configuration_template",

lib/galaxy/app_unittest_utils/galaxy_mock.py

@@ -329,7 +329,7 @@ def __init__(self, app=None, user=None, history=None, **kwargs):
         self.security = self.app.security
         self.history = history
 
-        self.request: Any = Bunch(headers={}, is_body_readable=False, host="request.host")
+        self.request: Any = Bunch(headers={}, is_body_readable=False, host="request.host", url_path="mock/url/path")
         self.response: Any = Bunch(headers={}, set_content_type=lambda i: None)
 
     @property

lib/galaxy/files/sources/dropbox.py

@@ -30,6 +30,8 @@ def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None)
         # accessToken has been renamed to access_token in fs.dropboxfs 1.0
         if "accessToken" in props:
             props["access_token"] = props.pop("accessToken")
+        if "oauth2_access_token" in props:
+            props["access_token"] = props.pop("oauth2_access_token")
 
         try:
             handle = DropboxFS(**{**props, **extra_props})

lib/galaxy/files/sources/googledrive.py

@@ -17,6 +17,9 @@ class GoogleDriveFilesSource(PyFilesystem2FilesSource):
 
     def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None):
         props = self._serialization_props(user_context)
+        access_token = props.pop("oauth2_access_token")
+        if access_token:
+            props["token"] = access_token
         credentials = Credentials(**props)
         handle = GoogleDriveFS(credentials)
         return handle

lib/galaxy/files/templates/__init__.py

@@ -4,6 +4,8 @@
     FileSourceTemplate,
     FileSourceTemplateSummaries,
     FileSourceTemplateType,
+    get_oauth2_config,
+    get_oauth2_config_or_none,
     template_to_configuration,
 )
 
@@ -13,5 +15,7 @@
     "FileSourceTemplate",
     "FileSourceTemplateSummaries",
     "FileSourceTemplateType",
+    "get_oauth2_config",
+    "get_oauth2_config_or_none",
     "template_to_configuration",
 )