fix

flexflow · Mar 13, 2024 · c856680 · c856680
1 parent e03dec0
commit c856680
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/inference/spec_infer/spec_infer.cc b/inference/spec_infer/spec_infer.cc
@@ -275,7 +275,7 @@ void FlexFlow::top_level_task(Task const *task,
   int max_requests_per_batch = 16;
   int max_tokens_per_batch = 256;
   int max_sequence_length = 1024;
-  int max_spec_tree_token_num = 20;
+  int max_spec_tree_token_num = 23;
   int expansion_degree = 3;
 
   InputArgs const &command_args = HighLevelRuntime::get_input_args();

diff --git a/src/runtime/request_manager.cc b/src/runtime/request_manager.cc
@@ -1577,9 +1577,11 @@ TreeVerifyBatchConfig RequestManager::prepare_next_batch_verify(
         }
 
         if (new_bc.num_tokens > get_max_verify_tokens_per_batch()) {
-          assert(false &&
-                 "Exceeding the space available in the TreeVerify batch");
-          break;
+          printf("Exceeding (%i) the space available (%i) in the TreeVerify "
+                 "batch\n",
+                 new_bc.num_tokens,
+                 get_max_verify_tokens_per_batch());
+          assert(false);
         }
 
         if (new_bc.requestsInfo[i].num_tokens_in_batch +