fix: embedding

Signed-off-by: thxCode <thxcode0824@gmail.com>
gpustack · Jul 29, 2024 · 632b63a · 632b63a
1 parent 7756804
commit 632b63a
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/llama-box/main.cpp b/llama-box/main.cpp
@@ -725,6 +725,9 @@ struct server_context {
     bool load_model(const llama_box_params &bparams) {
         params = bparams.gparams;
 
+        // always disable embedding for the server
+        params.embedding = false;
+
         // load multimodal projection model
         if (!params.mmproj.empty()) {
             if (params.n_ctx < 2048) {
@@ -2439,6 +2442,9 @@ struct server_context {
             return;
         }
 
+        // make sure we're in the right embedding mode
+        llama_set_embeddings(ctx, batch_type == 1);
+
         // process the created batch of tokens
         for (int32_t i = 0; i < batch.n_tokens; i += n_batch) {
             const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);