add step in checkpoint_dir to avoid name clashing

scripts/protein/esm2/esm2_pretrain.py

@@ -247,6 +247,7 @@ def main(
         save_top_k=save_top_k,
         every_n_train_steps=val_check_interval,
         always_save_context=True,  # Enables the .nemo file-like checkpointing where all IOMixins are under SerDe
+        filename="{epoch}-{step}-{val_loss:.2f}",
     )
 
     # Setup the logger and train the model

sub-packages/bionemo-example_model/tests/bionemo/example_model/lightning/test_lightning_basic.py

@@ -52,10 +52,11 @@ def _train_model_get_ckpt(
     checkpoint_callback = nl_callbacks.ModelCheckpoint(
         save_last=True,
         save_on_train_epoch_end=True,
-        monitor="reduced_train_loss",  # TODO find out how to get val_loss logged and use "val_loss",
+        monitor="val_loss",
         every_n_train_steps=5,
         always_save_context=True,  # Enables the .nemo file-like checkpointing where all IOMixins are under SerDe
         # async_save=False,  # Tries to save asynchronously, previously led to race conditions.
+        filename="{epoch}-{step}-{val_loss:.2f}"
     )
     save_dir = root_dir / name
     tb_logger = TensorBoardLogger(save_dir=save_dir, name=name)

sub-packages/bionemo-geneformer/src/bionemo/geneformer/scripts/train_geneformer.py

@@ -315,6 +315,7 @@ def main(
         save_top_k=save_top_k,
         every_n_train_steps=val_check_interval,
         always_save_context=True,  # Enables the .nemo file-like checkpointing where all IOMixins are under SerDe
+        # filename="{epoch}-{step}-{val_loss:.2f}",
     )
 
     # Setup the logger and train the model

sub-packages/bionemo-testing/src/bionemo/testing/harnesses/stop_and_go.py

@@ -219,10 +219,11 @@ def make_callbacks() -> Dict[Type[pl.Callback], pl.Callback]:
                 testing_callbacks.ValidLossCallback: testing_callbacks.ValidLossCallback(),
                 nl_callbacks.ModelCheckpoint: nl_callbacks.ModelCheckpoint(
                     save_last=True,
-                    monitor="reduced_train_loss",
+                    monitor="val_loss",
                     save_top_k=2,
                     every_n_train_steps=cls.val_check_interval,
                     always_save_context=True,
+                    filename="{epoch}-{step}-{val_loss:.2f}",
                 ),
             }