re-add prefix condition (#3183)

dust-tt · Jan 12, 2024 · f30be84 · f30be84
1 parent 90f05a1
commit f30be84
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/core/src/data_sources/splitter.rs b/core/src/data_sources/splitter.rs
@@ -213,6 +213,7 @@ impl TokenizedSection {
         if let Some(c) = content.as_ref() {
             if (c.tokens.len() + prefixes_tokens_count) > max_chunk_size
                 || !section.sections.is_empty()
+                || prefix.is_some()
             {
                 let effective_max_chunk_size = max_chunk_size - prefixes_tokens_count;