add liger to default config

anyscale · Nov 13, 2024 · 0a2853f · 0a2853f
1 parent 9369d94
commit 0a2853f
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/templates/e2e-dspy-workflow/README.md b/templates/e2e-dspy-workflow/README.md
@@ -378,8 +378,6 @@ sanity_check_program(llama_70b, vanilla_program, ft_trainset[0])
 ```
 
     Program input: Example({'text': 'I still have not received an answer as to why I was charged $1.00 in a transaction?'}) (input_keys={'text'})
-
-
     Program output label: extra_charge_on_statement
 
 
@@ -794,8 +792,6 @@ except ValueError as e:
 ```
 
     Program input: Example({'text': 'I still have not received an answer as to why I was charged $1.00 in a transaction?'}) (input_keys={'text'})
-
-
     Non fine-tuned model returned invalid output out and errored out with Expected dict_keys(['reasoning', 'label']) but got dict_keys([])
     Program input: Example({'text': 'I still have not received an answer as to why I was charged $1.00 in a transaction?'}) (input_keys={'text'})
     Fine-tuned model returned invalid output out and errored out with Expected dict_keys(['reasoning', 'label']) but got dict_keys([])

diff --git a/...s/fine-tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512.yaml b/...s/fine-tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512.yaml
@@ -41,6 +41,15 @@ deepspeed:
 worker_resources:
   anyscale/accelerator_shape:4xA10G: 0.001
 
+liger_kernel:
+  enabled: True
+  kwargs:
+    rms_norm: True
+    rope: True
+    swiglu: True
+    cross_entropy: True
+    fused_linear_cross_entropy: False
+
 # Lora configuration
 lora_config:
   r: 8