From 21cfa7a64ef4af775fb0e3629aea9ee5bb65dd97 Mon Sep 17 00:00:00 2001
From: Sebastien Flory <seb@dust.tt>
Date: Fri, 22 Nov 2024 08:48:01 +0100
Subject: [PATCH] Upg: increase chunks count batch per embedding request
 (8->128) (#8823)

---
 core/src/data_sources/data_source.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/data_sources/data_source.rs b/core/src/data_sources/data_source.rs
index 77c17a8afc0f..812c12204eb0 100644
--- a/core/src/data_sources/data_source.rs
+++ b/core/src/data_sources/data_source.rs
@@ -852,9 +852,9 @@ impl DataSource {
             .filter(|ci| !embeddings.contains_key(&ci.hash))
             .collect::<Vec<_>>();
 
-        // Chunk splits into a vectors of 8 chunks (Vec<Vec<String>>)
+        // Chunk splits into a vectors of 128 chunks (Vec<Vec<String>>)
         let chunked_splits = splits_to_embbed
-            .chunks(8)
+            .chunks(128)
             .map(|chunk| chunk.to_vec())
             .collect::<Vec<_>>();