seperate prefilling for increamental decoding

flexflow · Mar 29, 2024 · 4ef856f · 4ef856f
1 parent 0479a64
commit 4ef856f
Show file tree

Hide file tree

Showing 4 changed files with 930 additions and 371 deletions.
diff --git a/include/flexflow/request_manager.h b/include/flexflow/request_manager.h
@@ -63,6 +63,7 @@ struct Request {
     RUNNING = 102,   // running inference
     COMPLETED = 103, // finished and verified
     FINISHING = 104, // finishing request, but not yet verified
+    PREFILLING = 105 // prefilling the tree
   };
   BatchConfig::RequestGuid guid;
   int max_sequence_length;
@@ -307,6 +308,7 @@ class RequestManager {
   };
   std::unordered_map<RequestGuid, ProfileInfo> profiling_requests;
   double total_request_run_time;
+  BatchConfig buffer_bc = nullptr;
 };
 
 }; // namespace FlexFlow