[core] - feature: support eu-cluster-0 in Qdrant (#9625)

* [core] - feature: add support for new EUCluster0 in Qdrant data sources

 - Expand the QdrantCluster enum to include a new variant for EU-based cluster support
 - Update string conversions for cluster names to accommodate the new EU cluster
 - Include environment variable prefix handling for the new EUCluster0 configuration

* [front] - feature: use dynamic Qdrant cluster configuration from env variables

 - Replaced the hardcoded default Qdrant cluster with a dynamic one fetched from the environment configuration
 - Updated data source creation logic to utilize the new Qdrant cluster configuration method

[types] - refactor: support multiple Qdrant cluster options in type definitions

 - Expanded the `QdrantCluster` type to support an additional cluster called "eu-cluster-0"

* [front/lib/api] - refactor: adjust imports in config module

 - Separate type import from actual value import for better clarity

core/src/data_sources/qdrant.rs

@@ -23,6 +23,8 @@ use super::data_source::EmbedderConfig;
 pub enum QdrantCluster {
     #[serde(rename = "cluster-0")]
     Cluster0,
+    #[serde(rename = "eu-cluster-0")]
+    EUCluster0,
 }
 
 // See: https://www.notion.so/dust-tt/Design-Doc-Qdrant-re-arch-d0ebdd6ae8244ff593cdf10f08988c27
@@ -34,6 +36,7 @@ impl fmt::Display for QdrantCluster {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             QdrantCluster::Cluster0 => write!(f, "cluster-0"),
+            QdrantCluster::EUCluster0 => write!(f, "eu-cluster-0"),
         }
     }
 }
@@ -43,6 +46,7 @@ impl FromStr for QdrantCluster {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "cluster-0" => Ok(QdrantCluster::Cluster0),
+            "eu-cluster-0" => Ok(QdrantCluster::EUCluster0),
             _ => Err(ParseError::with_message("Unknown QdrantCluster"))?,
         }
     }
@@ -51,6 +55,7 @@ impl FromStr for QdrantCluster {
 pub fn env_var_prefix_for_cluster(cluster: QdrantCluster) -> &'static str {
     match cluster {
         QdrantCluster::Cluster0 => "QDRANT_CLUSTER_0",
+        QdrantCluster::EUCluster0 => "QDRANT_EU_CLUSTER_0",
     }
 }
 

front/lib/api/config.ts

@@ -1,3 +1,4 @@
+import type { QdrantCluster } from "@dust-tt/types";
 import { EnvironmentConfig } from "@dust-tt/types";
 
 export const PRODUCTION_DUST_API = "https://dust.tt";
@@ -161,6 +162,9 @@ const config = {
   getStatusPageApiToken: (): string => {
     return EnvironmentConfig.getEnvVariable("STATUS_PAGE_API_TOKEN");
   },
+  getQdrantCluster: (): QdrantCluster => {
+    return EnvironmentConfig.getEnvVariable("QDRANT_CLUSTER") as QdrantCluster;
+  },
 };
 
 export default config;

front/lib/api/data_sources.ts

@@ -25,7 +25,6 @@ import {
   ConnectorsAPI,
   CoreAPI,
   DEFAULT_EMBEDDING_PROVIDER_ID,
-  DEFAULT_QDRANT_CLUSTER,
   dustManagedCredentials,
   EMBEDDING_CONFIGS,
   Err,
@@ -850,7 +849,7 @@ export async function createDataSourceWithoutProvider(
     projectId: dustProject.value.project.project_id.toString(),
     config: {
       qdrant_config: {
-        cluster: DEFAULT_QDRANT_CLUSTER,
+        cluster: config.getQdrantCluster(),
         shadow_write_cluster: null,
       },
       embedder_config: {

front/pages/api/w/[wId]/data_sources/managed.ts

@@ -9,7 +9,6 @@ import {
   ConnectorsAPI,
   CoreAPI,
   DEFAULT_EMBEDDING_PROVIDER_ID,
-  DEFAULT_QDRANT_CLUSTER,
   dustManagedCredentials,
   EMBEDDING_CONFIGS,
   ioTsParsePayload,
@@ -277,7 +276,7 @@ async function handler(
             },
           },
           qdrant_config: {
-            cluster: DEFAULT_QDRANT_CLUSTER,
+            cluster: config.getQdrantCluster(),
             shadow_write_cluster: null,
           },
         },

front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/index.ts

@@ -11,7 +11,6 @@ import {
   ConnectorsAPI,
   CoreAPI,
   DEFAULT_EMBEDDING_PROVIDER_ID,
-  DEFAULT_QDRANT_CLUSTER,
   dustManagedCredentials,
   EMBEDDING_CONFIGS,
   ioTsParsePayload,
@@ -364,7 +363,7 @@ const handleDataSourceWithProvider = async ({
         },
       },
       qdrant_config: {
-        cluster: DEFAULT_QDRANT_CLUSTER,
+        cluster: config.getQdrantCluster(),
         shadow_write_cluster: null,
       },
     },

types/package.json

@@ -47,4 +47,4 @@
     "hot-shots": false,
     "child_process": false
   }
-}
+}

types/src/core/data_source.ts

@@ -1,5 +1,4 @@
-export type QdrantCluster = "cluster-0";
-export const DEFAULT_QDRANT_CLUSTER: QdrantCluster = "cluster-0";
+export type QdrantCluster = "cluster-0" | "eu-cluster-0";
 
 export interface EmbedderType {
   provider_id: string;