From 02a69746ca135a2e1a80b751fc4b7e05ea6182cf Mon Sep 17 00:00:00 2001
From: Adibvafa Fallahpour <90617686+Adibvafa@users.noreply.github.com>
Date: Thu, 25 Apr 2024 15:51:08 -0400
Subject: [PATCH] Update slurm_scripts.md

---
 slurm_scripts.md | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/slurm_scripts.md b/slurm_scripts.md
index 7c4d4e4..ab48416 100644
--- a/slurm_scripts.md
+++ b/slurm_scripts.md
@@ -79,6 +79,41 @@ stdbuf -oL -eL srun python3 finetune.py  \
 ```
 
 
+## BigBird - Pretrain
+```
+#!/bin/bash
+#SBATCH --job-name=bigbird_pretrain
+#SBATCH --gres=gpu:4
+#SBATCH --qos a100_amritk
+#SBATCH -p a100
+#SBATCH -c 24
+#SBATCH --time=23:00:00
+#SBATCH --mem=200G
+#SBATCH --output=/h/afallah/odyssey/multibird_a100-%j.out
+#SBATCH --error=/h/afallah/odyssey/multibird_a100-%j.err
+#SBATCH --no-requeue
+
+source /h/afallah/light/bin/activate
+
+cd /h/afallah/odyssey/odyssey
+
+export CUBLAS_WORKSPACE_CONFIG=:4096:2
+export NCCL_DEBUG=INFO
+export PYTHONFAULTHANDLER=1
+
+stdbuf -oL -eL srun python3 pretrain.py  \
+                --model-type cehr_bigbird \
+                --exp-name bigbird_pretrain \
+                --config-dir models/configs \
+                --data-dir data/bigbird_data \
+                --sequence-file patient_sequences/patient_sequences_2048.parquet \
+                --id-file patient_id_dict/dataset_2048_pretrain.pkl \
+                --vocab-dir data/vocab \
+                --val-size 0.1 \
+                --checkpoint-dir checkpoints/bigbird_pretrain
+```
+
+
 ## BigBird - Finetune Mortality
 ```
 #!/bin/bash