[Fix] Fix map_fn in custom_dataset/sft (#785)

InternLM · Jun 24, 2024 · 937e993 · 937e993
1 parent 72b645f
commit 937e993
Show file tree

Hide file tree

Showing 32 changed files with 64 additions and 64 deletions.
diff --git a/xtuner/configs/custom_dataset/sft/baichuan/baichuan2_13b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/baichuan/baichuan2_13b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/baichuan/baichuan2_7b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/baichuan/baichuan2_7b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/baichuan/baichuan_13b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/baichuan/baichuan_13b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/baichuan/baichuan_7b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/baichuan/baichuan_7b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/chatglm/chatglm2_6b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/chatglm/chatglm2_6b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/chatglm/chatglm3_6b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/chatglm/chatglm3_6b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -112,7 +112,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/deepseek/deepseek_moe_16b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/deepseek/deepseek_moe_16b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/...er/configs/custom_dataset/sft/deepseek/deepseekcoder_6_7b_instruct_qlora_custom_sft_e1.py b/...er/configs/custom_dataset/sft/deepseek/deepseekcoder_6_7b_instruct_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -115,7 +115,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/gemma/gemma_2b_it_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/gemma/gemma_2b_it_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/gemma/gemma_2b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/gemma/gemma_2b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/gemma/gemma_7b_it_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/gemma/gemma_7b_it_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/gemma/gemma_7b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/gemma/gemma_7b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_1_8b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_1_8b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_20b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_20b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_7b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/internlm/internlm2_chat_7b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/llama/llama2_70b_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/llama/llama2_70b_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -112,7 +112,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/llama/llama2_7b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/llama/llama2_7b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/mistral/mistral_7b_full_finetune_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/mistral/mistral_7b_full_finetune_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.dataset.samplers import InternRepoSampler
 from xtuner.engine import (DatasetInfoHook, EvaluateChatHook, ThroughputHook,
                            VarlenAttnArgsToMessageHubHook)
@@ -114,7 +114,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/mixtral/mixtral_8x7b_instruct_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/mixtral/mixtral_8x7b_instruct_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -114,7 +114,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/qwen/qwen1_5_0_5b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/qwen/qwen1_5_0_5b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/qwen/qwen1_5_14b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/qwen/qwen1_5_14b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,

diff --git a/xtuner/configs/custom_dataset/sft/qwen/qwen1_5_1_8b_chat_qlora_custom_sft_e1.py b/xtuner/configs/custom_dataset/sft/qwen/qwen1_5_1_8b_chat_qlora_custom_sft_e1.py
@@ -26,7 +26,7 @@
 
 from xtuner.dataset import process_hf_dataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import template_map_fn_factory
+from xtuner.dataset.map_fns import openai_map_fn, template_map_fn_factory
 from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook,
                                  VarlenAttnArgsToMessageHubHook)
 from xtuner.engine.runner import TrainLoop
@@ -111,7 +111,7 @@
     dataset=dict(type=load_dataset, path='json', data_files=data_files),
     tokenizer=tokenizer,
     max_length=max_length,
-    dataset_map_fn=None,
+    dataset_map_fn=openai_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     remove_unused_columns=True,