Multi datasource support (#311) (#316)

* added mds changes



* added mds changes



* added more changes



* upadted the group buttons and calls with mds



* added changes for and cache management



* added changes for PPL page, changes for fetching queries



* main test added



* added datsource dependency



* updated the tests for workbench with snapshots



* Update common/utils/fetch_datasources.ts




* updated lint checker in async workbench



* updated with new comments



* added UX changes



* added changed for pr comments



* added changes for linter



* added changes for snapshop



* added snapshot for main



* updated tests for main



---------

Signed-off-by: sumukhswamy <sumukhhs@amazon.com>
Signed-off-by: Sumukh Swamy <sumukhhs@amazon.com>
Co-authored-by: Joshua Li <joshuali925@gmail.com>

README.md

@@ -18,11 +18,11 @@ The OpenSearch Dashboards Query Workbench enables you to query your OpenSearch d
 [cypress-test-badge]: https://img.shields.io/badge/Cypress%20tests-in%20progress-yellow
 [cypress-test-link]: https://github.com/opensearch-project/opensearch-build/issues/1124
 [cypress-code-badge]: https://img.shields.io/badge/Cypress%20code-blue
-[cypress-code-link]: https://github.com/opensearch-project/sql/tree/main/workbench/.cypress/integration
+[cypress-code-link]: https://github.com/opensearch-project/dashboards-query-workbench/tree/main/.cypress/integration
 
 ## Documentation
 
-Please see our technical [documentation](https://opensearch.org/docs/latest/search-plugins/sql/workbench/) to learn more about its features.
+Please see our technical [documentation](https://opensearch.org/docs/latest/search-plugins/sql/workbench/) to learn more about its features. 
 
 
 ## Contributing

common/utils/async_query_helpers.ts

@@ -27,6 +27,7 @@ export const executeAsyncQuery = (
   currentDataSource: string,
   query: {},
   pollingCallback: PollingCallback,
+  dataSourceMDSId?: string,
   onErrorCallback?: (errorMessage: string) => void
 ) => {
   let jobId: string | undefined;
@@ -41,6 +42,7 @@ export const executeAsyncQuery = (
           ...query,
           sessionId: getAsyncSessionId(currentDataSource) ?? undefined,
         }),
+        query: { dataSourceMDSId },
       })
       .then((res) => {
         const responseData = res.data.resp;
@@ -76,7 +78,7 @@ export const executeAsyncQuery = (
   const pollQueryStatus = (id: string, callback: PollingCallback) => {
     !isQueryCancelled &&
       http
-        .get(ASYNC_QUERY_JOB_ENDPOINT + id)
+        .get(ASYNC_QUERY_JOB_ENDPOINT + id + `/` + dataSourceMDSId)
         .then((res: AsyncApiResponse) => {
           const status = res.data.resp.status.toLowerCase();
           const errorDetailsMessage = res.data.resp.error ?? '';

common/utils/fetch_datasources.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+import { CoreStart } from "../../../../src/core/public";
+
+export const fetchDataSources = (http: CoreStart['http'], dataSourceMDSId: string, urlDataSource: string, onSuccess, onError) => {
+    let dataOptions: { label: string; options?: Array<{ label: string }>; }[] = [];
+    let urlSourceFound = false;
+    if(!dataSourceMDSId){
+        dataSourceMDSId = ''
+    }
+
+    http.get(`/api/get_datasources/${dataSourceMDSId}`)
+      .then((res) => {
+        const data = res.data.resp;
+        const connectorGroups = {};
+
+        data.forEach((item) => {
+          const { connector, name } = item
+
+          if (connector === 'S3GLUE') {
+            if (!connectorGroups[connector]) {
+              connectorGroups[connector] = [];
+            }
+
+            connectorGroups[connector].push(name);
+            if (name === urlDataSource) {
+              urlSourceFound = true;
+            }
+          }
+        });
+
+        for (const connector in connectorGroups) {
+          if (connectorGroups.hasOwnProperty(connector)) {
+            const connectorNames = connectorGroups[connector];
+
+            dataOptions.push({
+              label: connector,
+              options: connectorNames.map((name) => ({ label: name })),
+            });
+          }
+        }
+
+        onSuccess(dataOptions, urlSourceFound);
+      })
+      .catch((err) => {
+        console.error(err);
+        onError(err);
+      });
+  };

opensearch_dashboards.json

@@ -4,12 +4,6 @@
   "opensearchDashboardsVersion": "2.14.0",
   "server": true,
   "ui": true,
-  "requiredPlugins": [
-    "navigation",
-    "opensearchDashboardsReact",
-    "opensearchDashboardsUtils"
-  ],
-  "optionalPlugins": [
-    "observabilityDashboards"
-  ]
+  "requiredPlugins": ["navigation", "opensearchDashboardsReact", "opensearchDashboardsUtils"],
+  "optionalPlugins": ["observabilityDashboards", "dataSource", "dataSourceManagement"]
 }

public/application.tsx

@@ -7,13 +7,15 @@
 import React from 'react';
 import ReactDOM from 'react-dom';
 import { AppMountParameters, CoreStart } from '../../../src/core/public';
-import { AppPluginStartDependencies } from './types';
+import { DataSourceManagementPluginSetup } from '../../../src/plugins/data_source_management/public';
 import { WorkbenchApp } from './components/app';
+import { AppPluginStartDependencies } from './types';
 
 export const renderApp = (
-  { notifications, http, chrome }: CoreStart,
-  { navigation }: AppPluginStartDependencies,
-  { appBasePath, element }: AppMountParameters
+  { notifications, http, chrome, savedObjects }: CoreStart,
+  { navigation, dataSource }: AppPluginStartDependencies,
+  { appBasePath, element, setHeaderActionMenu }: AppMountParameters,
+  dataSourceManagement: DataSourceManagementPluginSetup
 ) => {
   ReactDOM.render(
     <WorkbenchApp
@@ -22,6 +24,10 @@ export const renderApp = (
       http={http}
       navigation={navigation}
       chrome={chrome}
+      savedObjects={savedObjects}
+      dataSourceEnabled={!!dataSource}
+      dataSourceManagement={dataSourceManagement}
+      setActionMenu={setHeaderActionMenu}
     />,
     element
   );