Release: 0.3.3

setup.py

@@ -52,7 +52,7 @@
 # Setup
 setup(
     name="xllm",
-    version="0.3.2",
+    version="0.3.3",
     description="Simple & Cutting Edge LLM Finetuning",
     license_files=["LICENSE"],
     long_description=open("README.md", "r", encoding="utf-8").read(),

src/xllm/__init__.py

@@ -14,7 +14,7 @@
 
 # ruff: noqa: F401
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"
 
 from . import enums, types
 from .core.config import HuggingFaceConfig

src/xllm/core/config.py

@@ -398,6 +398,12 @@ class HuggingFaceConfig:
             "help": "Batch size on each GPU",
         },
     )
+    do_eval: bool = field(
+        default=False,
+        metadata={
+            "help": "Run eval or not",
+        },
+    )
     per_device_eval_batch_size: Optional[int] = field(
         default=None,
         metadata={
@@ -426,7 +432,7 @@ class HuggingFaceConfig:
         },
     )
     eval_steps: Union[int, float, None] = field(
-        default=None, metadata={"helps": "Number of update steps between two evaluations"}
+        default=1000, metadata={"helps": "Number of update steps between two evaluations"}
     )
     warmup_steps: int = field(
         default=1000,

src/xllm/core/dependencies.py

@@ -25,6 +25,7 @@
     AutoTokenizer,
     BitsAndBytesConfig,
     GPTQConfig,
+    IntervalStrategy,
     PreTrainedModel,
     PreTrainedTokenizer,
     Trainer,
@@ -54,16 +55,6 @@ def build_training_arguments(config: HuggingFaceConfig) -> TrainingArguments:
         fp16 = False
         bf16 = False
 
-    eval_kwargs = dict()
-
-    if config.eval_local_path_to_data is not None:
-        eval_kwargs["do_eval"] = True
-        eval_kwargs["evaluation_strategy"] = "steps"
-        eval_kwargs["per_device_eval_batch_size"] = config.per_device_eval_batch_size
-        eval_kwargs["eval_accumulation_steps"] = config.eval_accumulation_steps
-        eval_kwargs["eval_delay"] = config.eval_delay
-        eval_kwargs["eval_steps"] = config.eval_steps
-
     training_arguments = TrainingArguments(
         output_dir=config.output_dir,
         per_device_train_batch_size=config.per_device_train_batch_size,
@@ -95,7 +86,12 @@ def build_training_arguments(config: HuggingFaceConfig) -> TrainingArguments:
         disable_tqdm=False,
         logging_first_step=True,
         optim=config.optim,  # will be overwriten by deepspeed config if exist
-        **eval_kwargs,
+        do_eval=config.do_eval,
+        evaluation_strategy="steps" if config.do_eval else IntervalStrategy.NO,
+        per_device_eval_batch_size=config.per_device_eval_batch_size,
+        eval_accumulation_steps=config.eval_accumulation_steps,
+        eval_delay=config.eval_delay,
+        eval_steps=config.eval_steps,
     )
     return training_arguments
 

src/xllm/experiments/base.py

@@ -80,6 +80,14 @@ def internal_checks(self) -> None:
 
         return None
 
+    def internal_checks_before_train(self) -> None:
+        if self.training_arguments is not None and self.training_arguments.do_eval and self.eval_dataset is None:
+            raise ValueError(
+                f"You set do_eval at config to {self.config.do_eval}, "
+                "but experiment can't run eval, because eval_dataset is None. "
+                f"config.eval_local_path_to_data: {self.config.eval_local_path_to_data}"
+            )
+
     def build(self):
         dist_logger("Experiment building has started")
         self.at_beggining()
@@ -355,14 +363,15 @@ def before_train(self) -> None:
         return None
 
     def train(self):
+        self.before_train()
+
         if self.trainer is None:
             raise ValueError("trainer is None")
 
         if self.training_arguments is None:
             raise ValueError("training_arguments is None")
 
-        self.before_train()
-
+        self.internal_checks_before_train()
         dist_logger("Training will start soon")
         self.trainer.train()
         dist_logger("Training end")

tests/unit/experiments/test_base.py

@@ -30,6 +30,12 @@ def test_base_experiment_train(monkeypatch: MonkeyPatch, path_to_train_prepared_
         max_steps=2,
         tokenizer_name_or_path=LLAMA_TOKENIZER_DIR,
     )
+
+    # assert not config.do_eval
+    # assert
+    #
+    # print("HERE", config.do_eval,)
+
     with patch_from_pretrained_auto_causal_lm(monkeypatch=monkeypatch):
         experiment = Experiment(config=config)
         with patch_trainer_train(monkeypatch=monkeypatch):