From 8395f2cfe7985e488f02cbf8327e96d51f23ecad Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 16 Feb 2024 10:49:15 -0500 Subject: [PATCH 01/20] feat: experiment with Llama2 --- docs/Llama2FineTuning.md | 77 +++++++ jobs/Llama2/eval/job_original_use_7b_hf.sh | 21 ++ jobs/Llama2/eval/original_use_7b_hf.py | 31 +++ jobs/Llama2/finetune/finetune_7b_hf.py | 250 +++++++++++++++++++++ jobs/Llama2/finetune/job_finetune_7b_hf.sh | 30 +++ 5 files changed, 409 insertions(+) create mode 100644 docs/Llama2FineTuning.md create mode 100644 jobs/Llama2/eval/job_original_use_7b_hf.sh create mode 100644 jobs/Llama2/eval/original_use_7b_hf.py create mode 100644 jobs/Llama2/finetune/finetune_7b_hf.py create mode 100644 jobs/Llama2/finetune/job_finetune_7b_hf.sh diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md new file mode 100644 index 0000000..595bf36 --- /dev/null +++ b/docs/Llama2FineTuning.md @@ -0,0 +1,77 @@ +# Experiment with Llama2 model + +The experiment leveraged the [7B parameter Llama2 model pretrained by Meta](https://huggingface.co/meta-llama/Llama-2-7b-hf), +converted for the seamless use of the Hugging Face Transformers format. This model is choosen as a starting +point because it requires less training time and GPU resources compared to its larger counterparts, while it +potentially sacrifies some capability. Additionally, the Hugging Face Transformers format is selected because +of its extensive community support and standardized APIs. + +## Download Llama-2-7b-hf to Cedar + +1. Request access to Llama2 models on [the Meta website](https://llama.meta.com/llama-downloads/); +2. Followed the instructions on [the Hugging Face website](https://huggingface.co/meta-llama/Llama-2-7b-hf) +to request the access to its Llama2 model; +3. Request a hugging face access token on [this page](https://huggingface.co/settings/tokens); +4. Login to the Cedar cluster; +5. Create a "llama" directory and run these commands to download the model: + +``` +mkdir llama2 +cd llama2 + +// Load git-lfs first for downloading via Git large file storage +module load git-lfs/3.3.0 +git lfs install + +git clone https://{hugging_face_id}:{hugging_face_access_token}@huggingface.co/meta-llama/Llama-2-7b-hf + +// Fetch git large files in the repo directory +cd Llama-2-7b-hf +git lfs fetch +``` + +## Use the Llama2 model + +In the [`jobs/eval`](../jobs/Llama2/eval) directory, there are two scripts: + +* original_use_7b_hf.py: The script that loads the downloaded model and tokenizer to perform text generation, +word predictions and making inferences +* job_original_use_7b_hf.sh: The job script submitted to Cedar to run `original_use_7b_hf.py` + +Note that the job script must be copied to the user's `scratch` directory and is submitted from there using +the `sbatch` command. + +FTP scripts above to the cedar cluster in the users `llama2/eval` directory. Run the following command to +submit the job. + +``` +cp llama2/eval/job_original_use_7b_hf.sh scratch/. +cd scratch +sbatch job_original_use_7b_hf.sh +``` + +The result is written to the `llama2/eval/result.txt`. + +## Fine-tune the Llama2 model + +In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: + +* bliss.json: The dataset that converts English text to the structure in the Bliss language +* finetune_7b_hf.py: The script that fine-tunes the downloaded model +* job_finetune_7b_hf.sh: The job script submitted to Cedar to run `finetune_7b_hf.py` + +FTP scripts above to the cedar cluster in the users `llama2/finetune` directory. Run the following command to +submit the job. + +``` +cp llama2/eval/job_finetune_7b_hf.sh scratch/. +cd scratch +sbatch job_finetune_7b_hf.sh +``` + +## References + +[Llama2 in the Facebook Research Github repository](https://github.com/facebookresearch/llama) +[Llama2 fine-tune, inference examples](https://github.com/facebookresearch/llama-recipes) +[Llama2 on Hugging Face](https://huggingface.co/docs/transformers/model_doc/llama2) +[Use Hugging Face Models on Cedar Clusters](https://docs.alliancecan.ca/wiki/Huggingface) diff --git a/jobs/Llama2/eval/job_original_use_7b_hf.sh b/jobs/Llama2/eval/job_original_use_7b_hf.sh new file mode 100644 index 0000000..6af2120 --- /dev/null +++ b/jobs/Llama2/eval/job_original_use_7b_hf.sh @@ -0,0 +1,21 @@ +#!/bin/bash +#SBATCH --job-name=llama2-orig-use-7b-hf +#SBATCH --time 10-00:00 +#SBATCH --nodes=1 +#SBATCH --gpus-per-node=v100l:1 +#SBATCH --mem=128G +#SBATCH --ntasks-per-node=4 +#SBATCH --cpus-per-task=4 +#SBATCH --account=def-whkchun +#SBATCH --output=%x.o%j + +pip install --no-index --upgrade pip +module load python/3.8.2 +python -V + +source ~/llama2/.venv/bin/activate + +pip list + +echo "Llama2 original use from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." +python /home/cindyli/llama2/original_use7b_hf.py diff --git a/jobs/Llama2/eval/original_use_7b_hf.py b/jobs/Llama2/eval/original_use_7b_hf.py new file mode 100644 index 0000000..d47a659 --- /dev/null +++ b/jobs/Llama2/eval/original_use_7b_hf.py @@ -0,0 +1,31 @@ +from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline + +model_dir = "/home/cindyli/llama2/Llama-2-7b-hf" +tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True) +model = AutoModelForCausalLM.from_pretrained(model_dir, local_files_only=True) + +# # 1. Test prompt for text generation +# prompt = "What is a large language model?" +# pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=400) +# pipe_return = pipe(f"[INST] {prompt} [/INST]") +# # write results into a file +# file_path = "/home/cindyli/llama2/eval/result.txt" +# with open(file_path, "w") as file: +# file.write(f"Text generation: \n{pipe_return[0]['generated_text']}") + +# 2. Test prompt for word predictions +prompt = "David is feeling sick and in the hospital. He wants to" +pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer) +predictions = pipe(prompt, max_length=20, num_return_sequences=3) +file_path = "/home/cindyli/llama2/eval/result.txt" +with open(file_path, "w") as file: + for prediction in predictions: + file.write(f"- {prediction['generated_text']}: {prediction['score']}") + +# # 3. Test prompt for inference +# prompt = "David wants to" +# inference = tokenizer.decode(model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_length=300)[0]) +# # write results into a file +# file_path = "/home/cindyli/llama2/eval/result.txt" +# with open(file_path, "w") as file: +# file.write(f"Inference: \n{inference}") diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py new file mode 100644 index 0000000..375b971 --- /dev/null +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -0,0 +1,250 @@ +import os +import torch +from datasets import load_dataset +from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + BitsAndBytesConfig, + TrainingArguments, + pipeline, + logging, +) +from peft import LoraConfig, PeftModel +from trl import SFTTrainer + +# The local directory with the model and the tokenizer +model_dir = "/home/cindyli/llama2/Llama-2-7b-hf" + +# The instruction dataset to use +dataset_name = "/home/cindyli/llama2/finetune/bliss.json" + +# Output directory where the model predictions and checkpoints will be stored +output_dir = "/home/cindyli/llama2/results-finetune-7b-hf" + +# Fine-tuned model name +new_model = "llama-2-7b-hf-bliss" + +################################################################################ +# QLoRA parameters +################################################################################ + +# LoRA attention dimension +lora_r = 64 + +# Alpha parameter for LoRA scaling +lora_alpha = 16 + +# Dropout probability for LoRA layers +lora_dropout = 0.1 + +################################################################################ +# bitsandbytes parameters +################################################################################ + +# Activate 4-bit precision base model loading +use_4bit = True + +# Compute dtype for 4-bit base models +bnb_4bit_compute_dtype = "float16" + +# Quantization type (fp4 or nf4) +bnb_4bit_quant_type = "nf4" + +# Activate nested quantization for 4-bit base models (double quantization) +use_nested_quant = False + +################################################################################ +# TrainingArguments parameters +################################################################################ + +# Number of training epochs +num_train_epochs = 1 + +# Enable fp16/bf16 training (set bf16 to True with an A100) +fp16 = False +bf16 = False + +# Batch size per GPU for training +per_device_train_batch_size = 4 + +# Batch size per GPU for evaluation +per_device_eval_batch_size = 4 + +# Number of update steps to accumulate the gradients for +gradient_accumulation_steps = 1 + +# Enable gradient checkpointing +gradient_checkpointing = True + +# Maximum gradient normal (gradient clipping) +max_grad_norm = 0.3 + +# Initial learning rate (AdamW optimizer) +learning_rate = 2e-4 + +# Weight decay to apply to all layers except bias/LayerNorm weights +weight_decay = 0.001 + +# Optimizer to use +optim = "paged_adamw_32bit" + +# Learning rate schedule +lr_scheduler_type = "cosine" + +# Number of training steps (overrides num_train_epochs) +max_steps = -1 + +# Ratio of steps for a linear warmup (from 0 to learning rate) +warmup_ratio = 0.03 + +# Group sequences into batches with same length +# Saves memory and speeds up training considerably +group_by_length = True + +# Save checkpoint every X updates steps +save_steps = 0 + +# Log every X updates steps +logging_steps = 25 + +################################################################################ +# SFT parameters +################################################################################ + +# Maximum sequence length to use +max_seq_length = None + +# Pack multiple short examples in the same input sequence to increase efficiency +packing = False + +# Load the entire model on the GPU 0 +device_map = {"": 0} + +def create_prompt_formats(sample): + """ + Creates a formatted prompt template for an entry in the dataset + + :param sample: one dictionary from the dataset + """ + + # Initialize static strings for the prompt template + INTRO_BLURB = "Below is an example that converts an English sentence to a structure in the Bliss language." + INPUT_KEY = "Original English sentence:" + RESPONSE_KEY = "Sentence in the Bliss language structure:" + + # Combine a prompt with the static strings + blurb = f"{INTRO_BLURB}" + input_context = f"{INPUT_KEY}\n{sample['original']}" if sample["original"] else None + response = f"{RESPONSE_KEY}\n{sample['bliss']}" + + # Create a list of prompt template elements + parts = [part for part in [blurb, input_context, response] if part] + + # Join prompt template elements into a single string to create the prompt template + formatted_prompt = "\n\n".join(parts) + + # Store the formatted prompt template in a new key "text" + sample["text"] = formatted_prompt + + return sample + +# Load tokenizer and model with QLoRA configuration +compute_dtype = getattr(torch, bnb_4bit_compute_dtype) + +bnb_config = BitsAndBytesConfig( + load_in_4bit=use_4bit, + bnb_4bit_quant_type=bnb_4bit_quant_type, + bnb_4bit_compute_dtype=compute_dtype, + bnb_4bit_use_double_quant=use_nested_quant, +) + +# Check GPU compatibility with bfloat16 +if compute_dtype == torch.float16 and use_4bit: + major, _ = torch.cuda.get_device_capability() + if major >= 8: + print("=" * 80) + print("Your GPU supports bfloat16: accelerate training with bf16=True") + print("=" * 80) + +# Load base model +model = AutoModelForCausalLM.from_pretrained( + model_dir, + local_files_only=True, + quantization_config=bnb_config, + device_map=device_map +) +model.config.use_cache = False +model.config.pretraining_tp = 1 + +# Load LLaMA tokenizer +tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, trust_remote_code=True) +tokenizer.pad_token = tokenizer.eos_token +tokenizer.padding_side = "right" # Fix weird overflow issue with fp16 training + +# Preprocess dataset +print("Preprocessing dataset...") +dataset = load_dataset("json", data_files=dataset_name, split="train") +print(f"Number of prompts: {len(dataset)}") +print(f"Column names are: {dataset.column_names}") + +# Convert the data into prompts using the instructional template +dataset = dataset.map(create_prompt_formats) + +print(dataset) +print(dataset[0]) +print("Done with preprocessing dataset.\n\nStart fine-tuning...") + +# Load LoRA configuration +peft_config = LoraConfig( + lora_alpha=lora_alpha, + lora_dropout=lora_dropout, + r=lora_r, + bias="none", + task_type="CAUSAL_LM", +) + +# Set training parameters +training_arguments = TrainingArguments( + output_dir=output_dir, + num_train_epochs=num_train_epochs, + per_device_train_batch_size=per_device_train_batch_size, + gradient_accumulation_steps=gradient_accumulation_steps, + optim=optim, + save_steps=save_steps, + logging_steps=logging_steps, + learning_rate=learning_rate, + weight_decay=weight_decay, + fp16=fp16, + bf16=bf16, + max_grad_norm=max_grad_norm, + max_steps=max_steps, + warmup_ratio=warmup_ratio, + group_by_length=group_by_length, + lr_scheduler_type=lr_scheduler_type +) + +# Set supervised fine-tuning parameters +trainer = SFTTrainer( + model=model, + train_dataset=dataset, + peft_config=peft_config, + dataset_text_field="text", + max_seq_length=max_seq_length, + tokenizer=tokenizer, + args=training_arguments, + packing=packing, +) + +# Train model +trainer.train() + +# Save trained model +trainer.model.save_pretrained(new_model) + +print("Done with the fine-tuning.") + +# Run text generation pipeline with our next model +prompt = "Converts this sentence to a structure in the Bliss language: I am a software developer." +pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=200) +result = pipe(f"{prompt}") +print(result[0]['generated_text']) diff --git a/jobs/Llama2/finetune/job_finetune_7b_hf.sh b/jobs/Llama2/finetune/job_finetune_7b_hf.sh new file mode 100644 index 0000000..a5df7e4 --- /dev/null +++ b/jobs/Llama2/finetune/job_finetune_7b_hf.sh @@ -0,0 +1,30 @@ +#!/bin/bash +#SBATCH --job-name=llama2-finetune-7b-hf +#SBATCH --time 10-00:00 +#SBATCH --nodes=1 +#SBATCH --gpus-per-node=v100l:1 +#SBATCH --mem=128G +#SBATCH --ntasks-per-node=4 +#SBATCH --cpus-per-task=4 +#SBATCH --account=ctb-whkchun +#SBATCH --output=%x.o%j + +pip install --upgrade pip +module load python/3.11.5 +python -V + +virtualenv --no-download $SLURM_TMPDIR/env +source $SLURM_TMPDIR/.env/bin/activate + +pip install --upgrade pip + +module load StdEnv/2023 rust/1.70.0 arrow/14.0.1 gcc/12.3 +pip install --no-index transformers==4.36.2 accelerate==0.25.0 peft==0.5.0 bitsandbytes==0.40.2 +pip install datasets==2.17.0 trl +pip install -r /home/cindyli/llama2/requirements-llama2.txt + +python -V +pip list + +echo "Fine-tuning Llama2 from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." +python /home/cindyli/llama2/finetune/finetune_7b_hf.py From 4ba2365ef35cce2b09504f303b691e939c8e1357 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 16 Feb 2024 11:05:47 -0500 Subject: [PATCH 02/20] fix: linted --- jobs/Llama2/finetune/finetune_7b_hf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 375b971..42ea5a0 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -1,4 +1,3 @@ -import os import torch from datasets import load_dataset from transformers import ( @@ -6,10 +5,9 @@ AutoTokenizer, BitsAndBytesConfig, TrainingArguments, - pipeline, - logging, + pipeline ) -from peft import LoraConfig, PeftModel +from peft import LoraConfig from trl import SFTTrainer # The local directory with the model and the tokenizer @@ -120,6 +118,7 @@ # Load the entire model on the GPU 0 device_map = {"": 0} + def create_prompt_formats(sample): """ Creates a formatted prompt template for an entry in the dataset @@ -148,6 +147,7 @@ def create_prompt_formats(sample): return sample + # Load tokenizer and model with QLoRA configuration compute_dtype = getattr(torch, bnb_4bit_compute_dtype) @@ -179,7 +179,7 @@ def create_prompt_formats(sample): # Load LLaMA tokenizer tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, trust_remote_code=True) tokenizer.pad_token = tokenizer.eos_token -tokenizer.padding_side = "right" # Fix weird overflow issue with fp16 training +tokenizer.padding_side = "right" # Fix weird overflow issue with fp16 training # Preprocess dataset print("Preprocessing dataset...") From 860db14a21f4ce1e6ccab2f4d7c54cb89d7dc7ab Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 16 Feb 2024 13:56:22 -0500 Subject: [PATCH 03/20] fix: linted and clean up --- docs/Llama2FineTuning.md | 10 +++--- jobs/Llama2/eval/original_use_7b_hf.py | 31 ----------------- jobs/Llama2/finetune/finetune_7b_hf.py | 4 +-- .../job_original_use_7b_hf.sh | 2 +- .../Llama2/original_use/original_use_7b_hf.py | 33 +++++++++++++++++++ 5 files changed, 41 insertions(+), 39 deletions(-) delete mode 100644 jobs/Llama2/eval/original_use_7b_hf.py rename jobs/Llama2/{eval => original_use}/job_original_use_7b_hf.sh (87%) create mode 100644 jobs/Llama2/original_use/original_use_7b_hf.py diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index 595bf36..ef3860b 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -32,7 +32,7 @@ git lfs fetch ## Use the Llama2 model -In the [`jobs/eval`](../jobs/Llama2/eval) directory, there are two scripts: +In the [`jobs/original_use`](../jobs/Llama2/original_use) directory, there are two scripts: * original_use_7b_hf.py: The script that loads the downloaded model and tokenizer to perform text generation, word predictions and making inferences @@ -41,16 +41,16 @@ word predictions and making inferences Note that the job script must be copied to the user's `scratch` directory and is submitted from there using the `sbatch` command. -FTP scripts above to the cedar cluster in the users `llama2/eval` directory. Run the following command to +FTP scripts above to the cedar cluster in the users `llama2/original_use` directory. Run the following command to submit the job. ``` -cp llama2/eval/job_original_use_7b_hf.sh scratch/. +cp llama2/original_use/job_original_use_7b_hf.sh scratch/. cd scratch sbatch job_original_use_7b_hf.sh ``` -The result is written to the `llama2/eval/result.txt`. +The result is written to the `llama2/original_use/result.txt`. ## Fine-tune the Llama2 model @@ -64,7 +64,7 @@ FTP scripts above to the cedar cluster in the users `llama2/finetune` directory. submit the job. ``` -cp llama2/eval/job_finetune_7b_hf.sh scratch/. +cp llama2/finetune/job_finetune_7b_hf.sh scratch/. cd scratch sbatch job_finetune_7b_hf.sh ``` diff --git a/jobs/Llama2/eval/original_use_7b_hf.py b/jobs/Llama2/eval/original_use_7b_hf.py deleted file mode 100644 index d47a659..0000000 --- a/jobs/Llama2/eval/original_use_7b_hf.py +++ /dev/null @@ -1,31 +0,0 @@ -from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline - -model_dir = "/home/cindyli/llama2/Llama-2-7b-hf" -tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True) -model = AutoModelForCausalLM.from_pretrained(model_dir, local_files_only=True) - -# # 1. Test prompt for text generation -# prompt = "What is a large language model?" -# pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=400) -# pipe_return = pipe(f"[INST] {prompt} [/INST]") -# # write results into a file -# file_path = "/home/cindyli/llama2/eval/result.txt" -# with open(file_path, "w") as file: -# file.write(f"Text generation: \n{pipe_return[0]['generated_text']}") - -# 2. Test prompt for word predictions -prompt = "David is feeling sick and in the hospital. He wants to" -pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer) -predictions = pipe(prompt, max_length=20, num_return_sequences=3) -file_path = "/home/cindyli/llama2/eval/result.txt" -with open(file_path, "w") as file: - for prediction in predictions: - file.write(f"- {prediction['generated_text']}: {prediction['score']}") - -# # 3. Test prompt for inference -# prompt = "David wants to" -# inference = tokenizer.decode(model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_length=300)[0]) -# # write results into a file -# file_path = "/home/cindyli/llama2/eval/result.txt" -# with open(file_path, "w") as file: -# file.write(f"Inference: \n{inference}") diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 42ea5a0..cec53a7 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -16,8 +16,8 @@ # The instruction dataset to use dataset_name = "/home/cindyli/llama2/finetune/bliss.json" -# Output directory where the model predictions and checkpoints will be stored -output_dir = "/home/cindyli/llama2/results-finetune-7b-hf" +# Output directory where the model checkpoints will be stored +output_dir = "/home/cindyli/llama2/finetune/results-finetune-7b-hf" # Fine-tuned model name new_model = "llama-2-7b-hf-bliss" diff --git a/jobs/Llama2/eval/job_original_use_7b_hf.sh b/jobs/Llama2/original_use/job_original_use_7b_hf.sh similarity index 87% rename from jobs/Llama2/eval/job_original_use_7b_hf.sh rename to jobs/Llama2/original_use/job_original_use_7b_hf.sh index 6af2120..0a639b5 100644 --- a/jobs/Llama2/eval/job_original_use_7b_hf.sh +++ b/jobs/Llama2/original_use/job_original_use_7b_hf.sh @@ -18,4 +18,4 @@ source ~/llama2/.venv/bin/activate pip list echo "Llama2 original use from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." -python /home/cindyli/llama2/original_use7b_hf.py +python /home/cindyli/llama2/original_use/original_use_7b_hf.py diff --git a/jobs/Llama2/original_use/original_use_7b_hf.py b/jobs/Llama2/original_use/original_use_7b_hf.py new file mode 100644 index 0000000..d144e59 --- /dev/null +++ b/jobs/Llama2/original_use/original_use_7b_hf.py @@ -0,0 +1,33 @@ +from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline + +model_dir = "/home/cindyli/llama2/Llama-2-7b-hf" +tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True) +model = AutoModelForCausalLM.from_pretrained(model_dir, local_files_only=True) + +# 1. Test prompt for text generation +prompt = "What is a large language model?" +pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=400) +pipe_return = pipe(f"[INST] {prompt} [/INST]") + +# write results into the result file +file_path = "/home/cindyli/llama2/original_use/result.txt" +file = open(file_path, "w") +file.write(f"1. Text generation: \n{pipe_return[0]['generated_text']}\n\n") + +# 2. Test prompt for word predictions +prompt = "David is feeling sick and in the hospital. He wants to" +pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer) +predictions = pipe(prompt, max_length=20, num_return_sequences=3) + +# write results into the result file +file.write("2. Word prediction: \n") +for prediction in predictions: + file.write(f"- {prediction['generated_text']}: {prediction['score']}\n") +file.write("\n") + +# 3. Test prompt for inference +prompt = "David wants to" +inference = tokenizer.decode(model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_length=300)[0]) + +# write results into the result file +file.write(f"3. Inference: \n{inference}\n") From bdbb913446179975c33b608673b9c33381dbe4a1 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 16 Feb 2024 18:27:18 -0500 Subject: [PATCH 04/20] fix: update docs and scripts for the Llama2 experiment --- docs/Llama2FineTuning.md | 3 +++ .../original_use/job_original_use_7b_hf.sh | 17 +++++++++++++---- jobs/Llama2/original_use/original_use_7b_hf.py | 2 +- jobs/Llama2/requirements-llama2.txt | 4 ++++ 4 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 jobs/Llama2/requirements-llama2.txt diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index ef3860b..af4646b 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -30,6 +30,9 @@ cd Llama-2-7b-hf git lfs fetch ``` +6. Copy the content of [`requirements.txt`](https://github.com/facebookresearch/llama/blob/main/requirements.txt) +for setting up the Llama2 models into a new file named `requirements-llama2.txt` in the "llama" directory. + ## Use the Llama2 model In the [`jobs/original_use`](../jobs/Llama2/original_use) directory, there are two scripts: diff --git a/jobs/Llama2/original_use/job_original_use_7b_hf.sh b/jobs/Llama2/original_use/job_original_use_7b_hf.sh index 0a639b5..c2ff296 100644 --- a/jobs/Llama2/original_use/job_original_use_7b_hf.sh +++ b/jobs/Llama2/original_use/job_original_use_7b_hf.sh @@ -8,13 +8,22 @@ #SBATCH --cpus-per-task=4 #SBATCH --account=def-whkchun #SBATCH --output=%x.o%j - -pip install --no-index --upgrade pip -module load python/3.8.2 + +pip install --upgrade pip +module load python/3.11.5 python -V -source ~/llama2/.venv/bin/activate +virtualenv --no-download $SLURM_TMPDIR/env +source $SLURM_TMPDIR/.env/bin/activate + +pip install --upgrade pip +module load StdEnv/2023 +module load rust/1.70.0 +pip install --no-index transformers==4.36.2 +pip install -r /home/cindyli/llama2/requirements-llama2.txt + +python -V pip list echo "Llama2 original use from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." diff --git a/jobs/Llama2/original_use/original_use_7b_hf.py b/jobs/Llama2/original_use/original_use_7b_hf.py index d144e59..cb366da 100644 --- a/jobs/Llama2/original_use/original_use_7b_hf.py +++ b/jobs/Llama2/original_use/original_use_7b_hf.py @@ -22,7 +22,7 @@ # write results into the result file file.write("2. Word prediction: \n") for prediction in predictions: - file.write(f"- {prediction['generated_text']}: {prediction['score']}\n") + file.write(f"- {prediction['generated_text']}\n") file.write("\n") # 3. Test prompt for inference diff --git a/jobs/Llama2/requirements-llama2.txt b/jobs/Llama2/requirements-llama2.txt new file mode 100644 index 0000000..66f8a64 --- /dev/null +++ b/jobs/Llama2/requirements-llama2.txt @@ -0,0 +1,4 @@ +torch +fairscale +fire +sentencepiece From a8e35efceeb91f6c45fff1cb323dd10cf6d7983a Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 4 Mar 2024 09:56:25 -0500 Subject: [PATCH 05/20] fix: improve llama2 scripts and add fine-tuning results --- jobs/Llama2/finetune/bliss.json | 3870 +++++++++++++++++ jobs/Llama2/finetune/finetune_7b_hf.py | 129 +- jobs/Llama2/finetune/job_finetune_7b_hf.sh | 14 +- jobs/Llama2/finetune/results/7b-1epochs.txt | 328 ++ jobs/Llama2/finetune/results/7b-2epochs.txt | 342 ++ jobs/Llama2/finetune/results/7b-3epochs.txt | 348 ++ jobs/Llama2/finetune/results/7b-4epochs.txt | 360 ++ .../Llama2/original_use/original_use_7b_hf.py | 67 +- 8 files changed, 5388 insertions(+), 70 deletions(-) create mode 100644 jobs/Llama2/finetune/bliss.json create mode 100644 jobs/Llama2/finetune/results/7b-1epochs.txt create mode 100644 jobs/Llama2/finetune/results/7b-2epochs.txt create mode 100644 jobs/Llama2/finetune/results/7b-3epochs.txt create mode 100644 jobs/Llama2/finetune/results/7b-4epochs.txt diff --git a/jobs/Llama2/finetune/bliss.json b/jobs/Llama2/finetune/bliss.json new file mode 100644 index 0000000..4461469 --- /dev/null +++ b/jobs/Llama2/finetune/bliss.json @@ -0,0 +1,3870 @@ +[ + { + "original": "She will walk to school tomorrow morning.", + "bliss": "future:She walk to school tomorrow morning. " + }, + { + "original": "They will play soccer in the park tomorrow.", + "bliss": "future:They play soccer in park tomorrow. " + }, + { + "original": "He will visit his grandparents next weekend.", + "bliss": "future:He visit his grandparents weekend next. " + }, + { + "original": "We will cook dinner together tomorrow night.", + "bliss": "future:We cook together dinner tomorrow night. " + }, + { + "original": "I will finish reading the book tomorrow.", + "bliss": "future:I finish read book tomorrow. " + }, + { + "original": "The sun will set beautifully tomorrow evening.", + "bliss": "future:sun set beautifully tomorrow evening. " + }, + { + "original": "They will travel to Europe next summer.", + "bliss": "future:They travel to Europe summer next. " + }, + { + "original": "She will dance at the party next Friday.", + "bliss": "future:She dance at party Friday next. " + }, + { + "original": "He will study hard for his exams next month.", + "bliss": "future:He study hard for his exams month next. " + }, + { + "original": "We will visit the museum next week.", + "bliss": "future:We visit museum week next. " + }, + { + "original": "She is going to walk to school tomorrow morning.", + "bliss": "future:She walk to school tomorrow morning. " + }, + { + "original": "They are going to play soccer in the park tomorrow.", + "bliss": "future:They play soccer in park tomorrow. " + }, + { + "original": "He is going to visit his grandparents next weekend.", + "bliss": "future:He visit his grandparents weekend next. " + }, + { + "original": "We are going to cook dinner together tomorrow night.", + "bliss": "future:We cook dinner tomorrow night. " + }, + { + "original": "I am going to finish reading the book tomorrow.", + "bliss": "future:I finish read book tomorrow. " + }, + { + "original": "The sun is going to set beautifully tomorrow evening.", + "bliss": "future:sun set tomorrow evening. " + }, + { + "original": "They are going to travel to Europe next summer.", + "bliss": "future:They travel to Europe summer next. " + }, + { + "original": "She is going to dance at the party next Friday.", + "bliss": "future:She dance at party Friday next. " + }, + { + "original": "He is going to study hard for his exams next month.", + "bliss": "future:He study for his exams month next. " + }, + { + "original": "We are going to visit the museum next week.", + "bliss": "future:We visit museum week next. " + }, + { + "original": "She will eat breakfast tomorrow morning.", + "bliss": "future:She eat breakfast tomorrow morning. " + }, + { + "original": "They will go to the beach next summer.", + "bliss": "future:They go to beach summer next. " + }, + { + "original": "He will finish his homework before dinner.", + "bliss": "future:He finish his homework before dinner. " + }, + { + "original": "We will watch a movie together tomorrow night.", + "bliss": "future:We watch together movie tomorrow night. " + }, + { + "original": "I will visit my friend next weekend.", + "bliss": "future:I visit my friend weekend next. " + }, + { + "original": "The dog will chase the cat around the yard.", + "bliss": "future:dog chase cat around yard. " + }, + { + "original": "They will play soccer after school.", + "bliss": "future:They play soccer after school. " + }, + { + "original": "She will sing a beautiful song at the concert.", + "bliss": "future:She sing song beautiful at concert. " + }, + { + "original": "He will cook dinner for his family.", + "bliss": "future:He cook dinner for his family. " + }, + { + "original": "We will travel to Europe next year.", + "bliss": "future:We travel to Europe year next. " + }, + { + "original": "She is going to eat breakfast tomorrow morning.", + "bliss": "future:She eat breakfast tomorrow morning. " + }, + { + "original": "They are going to go to the beach next summer.", + "bliss": "future:They go to beach summer next. " + }, + { + "original": "He is going to finish his homework before dinner.", + "bliss": "future:He finish his homework before dinner. " + }, + { + "original": "We are going to watch a movie together tomorrow night.", + "bliss": "future:We watch movie tomorrow night. " + }, + { + "original": "I am going to visit my friend next weekend.", + "bliss": "future:I visit my friend weekend next. " + }, + { + "original": "The dog is going to chase the cat around the yard.", + "bliss": "future:dog chase cat around yard. " + }, + { + "original": "They are going to play soccer after school.", + "bliss": "future:They play soccer after school. " + }, + { + "original": "She is going to sing a beautiful song at the concert.", + "bliss": "future:She sing song beautiful at concert. " + }, + { + "original": "He is going to cook dinner for his family.", + "bliss": "future:He cook dinner for his family. " + }, + { + "original": "We are going to travel to Europe next year.", + "bliss": "future:We travel to Europe year next. " + }, + { + "original": "She will walk to school tomorrow.", + "bliss": "future:She walk to school tomorrow. " + }, + { + "original": "They will play soccer in the park tomorrow.", + "bliss": "future:They play soccer in park tomorrow. " + }, + { + "original": "He will visit his grandparents next weekend.", + "bliss": "future:He visit his grandparents weekend next. " + }, + { + "original": "We will cook dinner together tomorrow night.", + "bliss": "future:We cook together dinner tomorrow night. " + }, + { + "original": "I will finish reading the book tomorrow.", + "bliss": "future:I finish read book tomorrow. " + }, + { + "original": "The sun will rise early tomorrow morning.", + "bliss": "future:sun rise tomorrow morning. " + }, + { + "original": "They are going to travel to Europe next summer.", + "bliss": "future:They travel to Europe summer next. " + }, + { + "original": "She is going to dance at the party next Friday.", + "bliss": "future:She dance at party Friday next. " + }, + { + "original": "He is going to study hard for his exams next month.", + "bliss": "future:He study for his exams month next. " + }, + { + "original": "We are going to visit the museum next week.", + "bliss": "future:We visit museum week next. " + }, + { + "original": "She will wake up early tomorrow morning.", + "bliss": "future:She wake up tomorrow morning early. " + }, + { + "original": "They will play basketball tomorrow.", + "bliss": "future:They play basketball tomorrow. " + }, + { + "original": "He will visit his grandparents next week.", + "bliss": "future:He visit his grandparents week next. " + }, + { + "original": "We will finish our project by next month.", + "bliss": "future:We finish our project by month next. " + }, + { + "original": "I will travel to Paris next summer.", + "bliss": "future:I travel to Paris summer next. " + }, + { + "original": "The sun will set beautifully tomorrow evening.", + "bliss": "future:sun set beautifully tomorrow evening. " + }, + { + "original": "They will dance at the party next Friday.", + "bliss": "future:They dance at party Friday next. " + }, + { + "original": "She will study hard for her exams next month.", + "bliss": "future:She study hard for her exams month next. " + }, + { + "original": "He will cook dinner for his family tomorrow.", + "bliss": "future:He cook dinner for his family tomorrow. " + }, + { + "original": "We will go camping next year.", + "bliss": "future:We go camping year next. " + }, + { + "original": "She is going to wake up early tomorrow morning.", + "bliss": "future:She wake up tomorrow morning early. " + }, + { + "original": "They are going to play soccer in the park tomorrow.", + "bliss": "future:They play soccer in park tomorrow. " + }, + { + "original": "He is going to visit his grandparents next weekend.", + "bliss": "future:He visit his grandparents weekend next. " + }, + { + "original": "We are going to cook dinner together tomorrow night.", + "bliss": "future:We cook dinner tomorrow night. " + }, + { + "original": "I am going to finish reading the book tomorrow.", + "bliss": "future:I finish read book tomorrow. " + }, + { + "original": "The sun is going to shine brightly tomorrow.", + "bliss": "future:sun shine tomorrow. " + }, + { + "original": "They are going to travel to Europe next summer.", + "bliss": "future:They travel to Europe summer next. " + }, + { + "original": "She is going to dance at the party next Friday.", + "bliss": "future:She dance at party Friday next. " + }, + { + "original": "He is going to study hard for his exams next month.", + "bliss": "future:He study for his exams month next. " + }, + { + "original": "We are going to visit the museum next week.", + "bliss": "future:We visit museum week next. " + }, + { + "original": "She will study at the library tomorrow.", + "bliss": "future:She study at library tomorrow. " + }, + { + "original": "They will go for a hike next weekend.", + "bliss": "future:They go for hike weekend next. " + }, + { + "original": "He will finish his dinner early tomorrow night.", + "bliss": "future:He finish his dinner tomorrow night early. " + }, + { + "original": "We will visit the zoo next month.", + "bliss": "future:We visit zoo month next. " + }, + { + "original": "I will cook spaghetti for dinner tomorrow.", + "bliss": "future:I cook spaghetti for dinner tomorrow. " + }, + { + "original": "The movie will premiere next week.", + "bliss": "future:movie premiere week next. " + }, + { + "original": "They are planning to travel to Japan next summer.", + "bliss": "present progressive:They be plan to travel to Japan summer next. " + }, + { + "original": "She will dance at the party next Friday.", + "bliss": "future:She dance at party Friday next. " + }, + { + "original": "He will play the piano at the concert next week.", + "bliss": "future:He play piano at concert week next. " + }, + { + "original": "We will solve the puzzle tomorrow.", + "bliss": "future:We solve puzzle tomorrow. " + }, + { + "original": "She is going to wake up early tomorrow morning.", + "bliss": "future:She wake up tomorrow morning early. " + }, + { + "original": "They are going to play basketball at the park next weekend.", + "bliss": "future:They play basketball at park weekend next. " + }, + { + "original": "He is going to visit his aunt next week.", + "bliss": "future:He visit his aunt week next. " + }, + { + "original": "We are going to clean the house tomorrow.", + "bliss": "future:We clean house tomorrow. " + }, + { + "original": "I am going to watch a movie with my friends tomorrow evening.", + "bliss": "future:I watch movie with my friends tomorrow evening. " + }, + { + "original": "The sun is going to set beautifully tomorrow.", + "bliss": "future:sun set tomorrow. " + }, + { + "original": "They are going to have a barbecue in their backyard next weekend.", + "bliss": "future:They have in barbecue their backyard weekend next. " + }, + { + "original": "She is going to finish her homework tomorrow.", + "bliss": "future:She finish her homework tomorrow. " + }, + { + "original": "He is going to go for a run tomorrow morning.", + "bliss": "future:He go for run tomorrow morning. " + }, + { + "original": "We are going to attend a party next weekend.", + "bliss": "future:We attend party weekend next. " + }, + { + "original": "She will wake up early tomorrow morning.", + "bliss": "future:She wake up tomorrow morning early. " + }, + { + "original": "They will play basketball at the park tomorrow.", + "bliss": "future:They play basketball at park tomorrow. " + }, + { + "original": "He will visit his aunt tomorrow.", + "bliss": "future:He visit his aunt tomorrow. " + }, + { + "original": "We will clean the house tomorrow.", + "bliss": "future:We clean house tomorrow. " + }, + { + "original": "I will watch a movie with my friends tomorrow evening.", + "bliss": "future:I watch movie with my friends tomorrow evening. " + }, + { + "original": "The sun will set beautifully tomorrow.", + "bliss": "future:sun set beautifully tomorrow. " + }, + { + "original": "They will have a barbecue in their backyard tomorrow.", + "bliss": "future:They have barbecue in their backyard tomorrow. " + }, + { + "original": "She will finish her homework tomorrow.", + "bliss": "future:She finish her homework tomorrow. " + }, + { + "original": "He will go for a run tomorrow morning.", + "bliss": "future:He go for run tomorrow morning. " + }, + { + "original": "We will attend a party tomorrow night.", + "bliss": "future:We attend party tomorrow night. " + }, + { + "original": "She is going to wake up early tomorrow morning.", + "bliss": "future:She wake up tomorrow morning early. " + }, + { + "original": "They are going to play basketball at the park tomorrow.", + "bliss": "future:They play basketball at park tomorrow. " + }, + { + "original": "He is going to visit his aunt tomorrow.", + "bliss": "future:He visit his aunt tomorrow. " + }, + { + "original": "We are going to clean the house tomorrow.", + "bliss": "future:We clean house tomorrow. " + }, + { + "original": "I am going to watch a movie with my friends tomorrow evening.", + "bliss": "future:I watch movie with my friends tomorrow evening. " + }, + { + "original": "The sun is going to set beautifully tomorrow.", + "bliss": "future:sun set tomorrow. " + }, + { + "original": "They are going to have a barbecue in their backyard tomorrow.", + "bliss": "future:They have in barbecue their backyard tomorrow. " + }, + { + "original": "She is going to finish her homework tomorrow.", + "bliss": "future:She finish her homework tomorrow. " + }, + { + "original": "He is going to go for a run tomorrow morning.", + "bliss": "future:He go for run tomorrow morning. " + }, + { + "original": "We are going to attend a party tomorrow night.", + "bliss": "future:We attend party tomorrow night. " + }, + { + "original": "She will wake up early tomorrow morning.", + "bliss": "future:She wake up tomorrow morning early. " + }, + { + "original": "They will play basketball at the park tomorrow.", + "bliss": "future:They play basketball at park tomorrow. " + }, + { + "original": "He will visit his aunt tomorrow.", + "bliss": "future:He visit his aunt tomorrow. " + }, + { + "original": "We will clean the house tomorrow.", + "bliss": "future:We clean house tomorrow. " + }, + { + "original": "I will watch a movie with my friends tomorrow evening.", + "bliss": "future:I watch movie with my friends tomorrow evening. " + }, + { + "original": "The sun will set beautifully tomorrow.", + "bliss": "future:sun set beautifully tomorrow. " + }, + { + "original": "They will have a barbecue in their backyard tomorrow.", + "bliss": "future:They have barbecue in their backyard tomorrow. " + }, + { + "original": "She will finish her homework tomorrow.", + "bliss": "future:She finish her homework tomorrow. " + }, + { + "original": "He will go for a run tomorrow morning.", + "bliss": "future:He go for run tomorrow morning. " + }, + { + "original": "We will attend a party tomorrow night.", + "bliss": "future:We attend party tomorrow night. " + }, + { + "original": "She will wake up early tomorrow morning.", + "bliss": "future:She wake up tomorrow morning early. " + }, + { + "original": "They will play basketball at the park tomorrow.", + "bliss": "future:They play basketball at park tomorrow. " + }, + { + "original": "He will visit his aunt tomorrow.", + "bliss": "future:He visit his aunt tomorrow. " + }, + { + "original": "We will clean the house tomorrow.", + "bliss": "future:We clean house tomorrow. " + }, + { + "original": "I will watch a movie with my friends tomorrow evening.", + "bliss": "future:I watch movie with my friends tomorrow evening. " + }, + { + "original": "The sun will set beautifully tomorrow.", + "bliss": "future:sun set beautifully tomorrow. " + }, + { + "original": "They will have a barbecue in their backyard tomorrow.", + "bliss": "future:They have barbecue in their backyard tomorrow. " + }, + { + "original": "She will finish her homework tomorrow.", + "bliss": "future:She finish her homework tomorrow. " + }, + { + "original": "He will go for a run tomorrow morning.", + "bliss": "future:He go for run tomorrow morning. " + }, + { + "original": "We will attend a party tomorrow night.", + "bliss": "future:We attend party tomorrow night. " + }, + { + "original": "She will walk to school tomorrow.", + "bliss": "future:She walk to school tomorrow. " + }, + { + "original": "He will eat breakfast early tomorrow morning.", + "bliss": "future:He eat breakfast tomorrow morning. " + }, + { + "original": "They will watch a movie tomorrow night.", + "bliss": "future:They watch movie tomorrow night. " + }, + { + "original": "I will visit my grandparents next weekend.", + "bliss": "future:I visit my grandparents weekend next. " + }, + { + "original": "We will play soccer after school tomorrow.", + "bliss": "future:We play soccer after school tomorrow. " + }, + { + "original": "The sun will shine brightly tomorrow.", + "bliss": "future:sun shine brightly tomorrow. " + }, + { + "original": "He will finish his homework before dinner tomorrow.", + "bliss": "future:He finish his homework before dinner tomorrow. " + }, + { + "original": "They will go swimming at the beach next summer.", + "bliss": "future:They go swim at beach summer next. " + }, + { + "original": "She will cook dinner for her family tomorrow.", + "bliss": "future:She cook dinner for her family tomorrow. " + }, + { + "original": "The cat will sleep all day tomorrow.", + "bliss": "future:cat sleep day tomorrow. " + }, + { + "original": "He will run five miles tomorrow morning.", + "bliss": "future:He run miles tomorrow morning. " + }, + { + "original": "They will clean the house next weekend.", + "bliss": "future:They clean house weekend next. " + }, + { + "original": "I will read a book before going to bed tomorrow night.", + "bliss": "future:I read book before go to bed tomorrow night. " + }, + { + "original": "We will visit the museum next month.", + "bliss": "future:We visit museum month next. " + }, + { + "original": "She will buy a new dress next week.", + "bliss": "future:She buy dress new week next. " + }, + { + "original": "He will ride his bike to work tomorrow.", + "bliss": "future:He ride his bike to work tomorrow. " + }, + { + "original": "They will dance at the party tomorrow night.", + "bliss": "future:They dance at party tomorrow night. " + }, + { + "original": "I will graduate from college in the future.", + "bliss": "future:I graduate from college in future. " + }, + { + "original": "We will go on a road trip next summer.", + "bliss": "future:We go on road trip summer next. " + }, + { + "original": "She will paint a beautiful picture tomorrow night.", + "bliss": "future:She paint picture beautiful tomorrow night. " + }, + { + "original": "He will fix the broken window next weekend.", + "bliss": "future:He fix window broken weekend next. " + }, + { + "original": "They will sing songs around the campfire tomorrow night.", + "bliss": "future:They sing songs around campfire tomorrow night. " + }, + { + "original": "I will wake up early tomorrow morning.", + "bliss": "future:I wake up tomorrow morning early. " + }, + { + "original": "We will play board games all night tomorrow night.", + "bliss": "future:We play board games night tomorrow night. " + }, + { + "original": "She will bake cookies for her friends tomorrow.", + "bliss": "future:She bake cookies for her friends tomorrow. " + }, + { + "original": "He will travel to Europe next summer.", + "bliss": "future:He travel to Europe summer next. " + }, + { + "original": "They will attend a concert tomorrow night.", + "bliss": "future:They attend concert tomorrow night. " + }, + { + "original": "I will finish reading the book next week.", + "bliss": "future:I finish read book week next. " + }, + { + "original": "We will go fishing at the lake next weekend.", + "bliss": "future:We go fish at lake weekend next. " + }, + { + "original": "She will practice the piano for hours tomorrow.", + "bliss": "future:She practice piano for hours tomorrow. " + }, + { + "original": "He will clean his room tomorrow.", + "bliss": "future:He clean his room tomorrow. " + }, + { + "original": "They will plant flowers in the garden next spring.", + "bliss": "future:They plant flowers in garden spring next. " + }, + { + "original": "I will watch a documentary on TV tomorrow night.", + "bliss": "future:I watch on documentary TV tomorrow night. " + }, + { + "original": "We will have a barbecue in the backyard next weekend.", + "bliss": "future:We have barbecue in backyard weekend next. " + }, + { + "original": "She will study for her exams next week.", + "bliss": "future:She study for her exams week next. " + }, + { + "original": "He will fix the leaky faucet next weekend.", + "bliss": "future:He fix faucet leaky weekend next. " + }, + { + "original": "They will walk in the park tomorrow.", + "bliss": "future:They walk in park tomorrow. " + }, + { + "original": "I will play basketball with my friends tomorrow night.", + "bliss": "future:I play basketball with my friends tomorrow night. " + }, + { + "original": "We will visit the zoo next weekend.", + "bliss": "future:We visit zoo weekend next. " + }, + { + "original": "She will write a letter to her pen pal tomorrow.", + "bliss": "future:She write letter to her pen pal tomorrow. " + }, + { + "original": "He will mow the lawn next week.", + "bliss": "future:He mow lawn week next. " + }, + { + "original": "They will go shopping at the mall next weekend.", + "bliss": "future:They go shop at mall weekend next. " + }, + { + "original": "I will go for a run tomorrow morning.", + "bliss": "future:I go for run tomorrow morning. " + }, + { + "original": "We will watch the sunset at the beach tomorrow night.", + "bliss": "future:We watch sunset at beach tomorrow night. " + }, + { + "original": "She will finish knitting a sweater next week.", + "bliss": "future:She finish knit sweater week next. " + }, + { + "original": "He will cook dinner for his family tomorrow.", + "bliss": "future:He cook dinner for his family tomorrow. " + }, + { + "original": "They will hike in the mountains next weekend.", + "bliss": "future:They hike in mountains weekend next. " + }, + { + "original": "I will clean out the garage next weekend.", + "bliss": "future:I clean out garage weekend next. " + }, + { + "original": "We will have a picnic in the park next weekend.", + "bliss": "future:We have in picnic park weekend next. " + }, + { + "original": "She will paint her nails tomorrow.", + "bliss": "future:She paint her nails tomorrow. " + }, + { + "original": "He will read a book in the garden tomorrow.", + "bliss": "future:He read book in garden tomorrow. " + }, + { + "original": "They will play soccer in the park tomorrow.", + "bliss": "future:They play soccer in park tomorrow. " + }, + { + "original": "I will attend a party next weekend.", + "bliss": "future:I attend party weekend next. " + }, + { + "original": "We will ride bikes around the neighborhood next weekend.", + "bliss": "future:We ride around bikes neighborhood weekend next. " + }, + { + "original": "She will watch a movie with her friends tomorrow night.", + "bliss": "future:She watch movie with her friends tomorrow night. " + }, + { + "original": "He will go for a swim at the pool tomorrow.", + "bliss": "future:He go for at swim pool tomorrow. " + }, + { + "original": "They will go for a hike in the forest tomorrow.", + "bliss": "future:They go for in hike forest tomorrow. " + }, + { + "original": "I will visit my aunt next weekend.", + "bliss": "future:I visit my aunt weekend next. " + }, + { + "original": "We will play video games all day tomorrow.", + "bliss": "future:We play video games day tomorrow. " + }, + { + "original": "She will cook a delicious meal tomorrow.", + "bliss": "future:She cook meal delicious tomorrow. " + }, + { + "original": "He will ride his skateboard to the park tomorrow.", + "bliss": "future:He ride his skateboard to park tomorrow. " + }, + { + "original": "They will have a barbecue in the backyard tomorrow.", + "bliss": "future:They have barbecue in backyard tomorrow. " + }, + { + "original": "I will go camping with my family next weekend.", + "bliss": "future:I go camp with my family weekend next. " + }, + { + "original": "We will go to the beach next weekend.", + "bliss": "future:We go to beach weekend next. " + }, + { + "original": "She will practice yoga in the morning tomorrow.", + "bliss": "future:She practice yoga in morning tomorrow. " + }, + { + "original": "He will listen to music in his room tomorrow.", + "bliss": "future:He listen to music in his room tomorrow. " + }, + { + "original": "They will have a sleepover at their friend's house next weekend.", + "bliss": "future:They have sleepover at their friend house weekend next. " + }, + { + "original": "I will go for a bike ride tomorrow.", + "bliss": "future:I go for bike ride tomorrow. " + }, + { + "original": "We will go for a walk in the park tomorrow.", + "bliss": "future:We go for in walk park tomorrow. " + }, + { + "original": "She will take a nap in the afternoon tomorrow.", + "bliss": "future:She take nap in afternoon tomorrow. " + }, + { + "original": "He will play guitar for his friends tomorrow.", + "bliss": "future:He play guitar for his friends tomorrow. " + }, + { + "original": "They will go for a drive in the countryside tomorrow.", + "bliss": "future:They go for in drive countryside tomorrow. " + }, + { + "original": "I will visit the library next week.", + "bliss": "future:I visit library week next. " + }, + { + "original": "We will go to a party tomorrow night.", + "bliss": "future:We go to party tomorrow night. " + }, + { + "original": "She will swim in the pool tomorrow.", + "bliss": "future:She swim in pool tomorrow. " + }, + { + "original": "He will cook breakfast for his family tomorrow.", + "bliss": "future:He cook breakfast for his family tomorrow. " + }, + { + "original": "They will watch a movie at home tomorrow.", + "bliss": "future:They watch movie at home tomorrow. " + }, + { + "original": "I will play video games with my brother tomorrow.", + "bliss": "future:I play video games with my brother tomorrow. " + }, + { + "original": "We will go to the amusement park next weekend.", + "bliss": "future:We go to amusement park weekend next. " + }, + { + "original": "She will go shopping with her friends tomorrow.", + "bliss": "future:She go shop with her friends tomorrow. " + }, + { + "original": "He will ride his bike to school tomorrow.", + "bliss": "future:He ride his bike to school tomorrow. " + }, + { + "original": "They will visit their grandparents next weekend.", + "bliss": "future:They visit their grandparents weekend next. " + }, + { + "original": "I will clean my room tomorrow.", + "bliss": "future:I clean my room tomorrow. " + }, + { + "original": "We will have a picnic in the backyard next weekend.", + "bliss": "future:We have in picnic backyard weekend next. " + }, + { + "original": "She will read a book in the garden tomorrow.", + "bliss": "future:She read book in garden tomorrow. " + }, + { + "original": "He will play soccer with his friends tomorrow.", + "bliss": "future:He play soccer with his friends tomorrow. " + }, + { + "original": "They will have a barbecue at the beach tomorrow.", + "bliss": "future:They have barbecue at beach tomorrow. " + }, + { + "original": "I will go for a swim in the lake tomorrow.", + "bliss": "future:I go for in swim lake tomorrow. " + }, + { + "original": "We will watch the sunset at the park next weekend.", + "bliss": "future:We watch sunset at park weekend next. " + }, + { + "original": "She will go hiking with her friends tomorrow.", + "bliss": "future:She go hike with her friends tomorrow. " + }, + { + "original": "He will ride his skateboard at the skate park tomorrow.", + "bliss": "future:He ride his skateboard at skate park tomorrow. " + }, + { + "original": "They will have a picnic in the forest tomorrow.", + "bliss": "future:They have in picnic forest tomorrow. " + }, + { + "original": "I will visit the museum next weekend.", + "bliss": "future:I visit museum weekend next. " + }, + { + "original": "We will go to a concert tomorrow night.", + "bliss": "future:We go to concert tomorrow night. " + }, + { + "original": "She will play tennis with her friends tomorrow.", + "bliss": "future:She play tennis with her friends tomorrow. " + }, + { + "original": "He will go fishing with his dad tomorrow.", + "bliss": "future:He go fish with his dad tomorrow. " + }, + { + "original": "They will have a barbecue in the park tomorrow.", + "bliss": "future:They have in barbecue park tomorrow. " + }, + { + "original": "I will go for a jog in the morning tomorrow.", + "bliss": "future:I go for in jog morning tomorrow. " + }, + { + "original": "We will have a bonfire at the beach next weekend.", + "bliss": "future:We have at bonfire beach weekend next. " + }, + { + "original": "She will go for a bike ride in the countryside tomorrow.", + "bliss": "future:She go for bike in ride countryside tomorrow. " + }, + { + "original": "She will walk to the store tomorrow.", + "bliss": "future:She walk to store tomorrow. " + }, + { + "original": "They will play soccer after school.", + "bliss": "future:They play soccer after school. " + }, + { + "original": "He will study for his exam all night.", + "bliss": "future:He study for his exam night. " + }, + { + "original": "We will visit Paris next summer.", + "bliss": "future:We visit Paris summer next. " + }, + { + "original": "The cat will chase the mouse around the house.", + "bliss": "future:cat chase mouse around house. " + }, + { + "original": "I will finish my homework before dinner.", + "bliss": "future:I finish my homework before dinner. " + }, + { + "original": "She will cook dinner for her family.", + "bliss": "future:She cook dinner for her family. " + }, + { + "original": "They will watch a movie at the cinema.", + "bliss": "future:They watch movie at cinema. " + }, + { + "original": "He will run a marathon next month.", + "bliss": "future:He run marathon month next. " + }, + { + "original": "We will go camping in the mountains.", + "bliss": "future:We go in camping mountains. " + }, + { + "original": "The sun will set over the horizon.", + "bliss": "future:sun set over horizon. " + }, + { + "original": "I will read a book in bed tomorrow night.", + "bliss": "future:I read book in bed tomorrow night. " + }, + { + "original": "She will paint her bedroom walls blue.", + "bliss": "future:She paint her bedroom walls. " + }, + { + "original": "They will visit their grandparents over the holidays.", + "bliss": "future:They visit their grandparents over holidays. " + }, + { + "original": "He will fix the leaky faucet in the bathroom.", + "bliss": "future:He fix faucet leaky in bathroom. " + }, + { + "original": "We will ride bikes through the park.", + "bliss": "future:We ride bikes through park. " + }, + { + "original": "The children will play in the backyard.", + "bliss": "future:children play in backyard. " + }, + { + "original": "She will dance at her sister's wedding.", + "bliss": "future:She dance at her sister wedding. " + }, + { + "original": "They will build a sandcastle at the beach.", + "bliss": "future:They build at sandcastle beach. " + }, + { + "original": "He will wake up early tomorrow morning.", + "bliss": "future:He wake up tomorrow morning early. " + }, + { + "original": "I will eat breakfast before leaving for work.", + "bliss": "future:I eat breakfast before leave for work. " + }, + { + "original": "She will swim in the pool for hours.", + "bliss": "future:She swim in pool for hours. " + }, + { + "original": "They will visit the museum downtown.", + "bliss": "future:They visit museum downtown. " + }, + { + "original": "We will hike up the mountain trail.", + "bliss": "future:We hike up mountain trail. " + }, + { + "original": "The dog will bark at the mailman.", + "bliss": "future:dog bark at mailman. " + }, + { + "original": "I will drive to the grocery store.", + "bliss": "future:I drive to grocery store. " + }, + { + "original": "She will sing a song at the talent show.", + "bliss": "future:She sing song at talent show. " + }, + { + "original": "They will fly to Europe for vacation.", + "bliss": "future:They fly to Europe for vacation. " + }, + { + "original": "He will cook dinner on the grill.", + "bliss": "future:He cook dinner on grill. " + }, + { + "original": "We will watch fireworks on the Fourth of July.", + "bliss": "future:We watch fireworks on of Fourth July. " + }, + { + "original": "The baby will sleep through the night.", + "bliss": "future:baby sleep through night. " + }, + { + "original": "I will complete the project ahead of schedule.", + "bliss": "future:I complete ahead project of schedule. " + }, + { + "original": "She will knit a sweater for her friend.", + "bliss": "future:She knit sweater for her friend. " + }, + { + "original": "They will take a road trip across the country.", + "bliss": "future:They take road across trip country. " + }, + { + "original": "He will climb to the top of the mountain.", + "bliss": "future:He climb to of top mountain. " + }, + { + "original": "We will explore the ancient ruins.", + "bliss": "future:We explore ruins ancient. " + }, + { + "original": "The train will arrive at the station.", + "bliss": "future:train arrive at station. " + }, + { + "original": "I will finish reading the novel.", + "bliss": "future:I finish read novel. " + }, + { + "original": "She will write a letter to her pen pal.", + "bliss": "future:She write letter to her pen pal. " + }, + { + "original": "They will visit the zoo on Saturday.", + "bliss": "future:They visit zoo on Saturday. " + }, + { + "original": "He will fix the broken window.", + "bliss": "future:He fix window broken. " + }, + { + "original": "We will play board games by the fireplace.", + "bliss": "future:We play board games by fireplace. " + }, + { + "original": "The birds will chirp in the trees.", + "bliss": "future:birds chirp in trees. " + }, + { + "original": "I will ride my bike to school.", + "bliss": "future:I ride my bike to school. " + }, + { + "original": "She will sew a dress for the school play.", + "bliss": "future:She sew for dress school play. " + }, + { + "original": "They will plant flowers in the garden.", + "bliss": "future:They plant flowers in garden. " + }, + { + "original": "He will visit his old high school.", + "bliss": "future:He visit his school old high. " + }, + { + "original": "We will cook dinner together as a family.", + "bliss": "future:We cook together dinner as family. " + }, + { + "original": "The rain will pour down all afternoon.", + "bliss": "future:rain pour down afternoon. " + }, + { + "original": "I will bake cookies for the bake sale.", + "bliss": "future:I bake cookies for bake sale. " + }, + { + "original": "She will walk to the store tomorrow.", + "bliss": "future:She walk to store tomorrow. " + }, + { + "original": "They will play soccer after school.", + "bliss": "future:They play soccer after school. " + }, + { + "original": "He will study for his exam all night.", + "bliss": "future:He study for his exam night. " + }, + { + "original": "We will visit our grandparents next weekend.", + "bliss": "future:We visit our grandparents weekend next. " + }, + { + "original": "The cat will chase the mouse around the house.", + "bliss": "future:cat chase mouse around house. " + }, + { + "original": "I will finish my homework before dinner.", + "bliss": "future:I finish my homework before dinner. " + }, + { + "original": "She will cook dinner for her family.", + "bliss": "future:She cook dinner for her family. " + }, + { + "original": "They will watch a movie at the cinema.", + "bliss": "future:They watch movie at cinema. " + }, + { + "original": "He will run a marathon next month.", + "bliss": "future:He run marathon month next. " + }, + { + "original": "We will go swimming at the beach next summer.", + "bliss": "future:We go swim at beach summer next. " + }, + { + "original": "The sun will set over the horizon.", + "bliss": "future:sun set over horizon. " + }, + { + "original": "I will read a book in bed tomorrow night.", + "bliss": "future:I read book in bed tomorrow night. " + }, + { + "original": "She will paint her bedroom walls blue.", + "bliss": "future:She paint her bedroom walls. " + }, + { + "original": "They will travel to Europe for vacation.", + "bliss": "future:They travel to Europe for vacation. " + }, + { + "original": "He will fix the leaky faucet in the bathroom.", + "bliss": "future:He fix faucet leaky in bathroom. " + }, + { + "original": "We will hike up the mountain trail.", + "bliss": "future:We hike up mountain trail. " + }, + { + "original": "The children will play in the backyard.", + "bliss": "future:children play in backyard. " + }, + { + "original": "She will dance at her sister's wedding.", + "bliss": "future:She dance at her sister wedding. " + }, + { + "original": "They will build a sandcastle at the beach.", + "bliss": "future:They build at sandcastle beach. " + }, + { + "original": "He will wake up early tomorrow morning.", + "bliss": "future:He wake up tomorrow morning early. " + }, + { + "original": "I will take a bus to work tomorrow.", + "bliss": "future:I take bus to work tomorrow. " + }, + { + "original": "She will swim in the pool for hours.", + "bliss": "future:She swim in pool for hours. " + }, + { + "original": "They will visit the museum downtown.", + "bliss": "future:They visit museum downtown. " + }, + { + "original": "We will explore the ancient ruins.", + "bliss": "future:We explore ruins ancient. " + }, + { + "original": "The dog will bark at the mailman.", + "bliss": "future:dog bark at mailman. " + }, + { + "original": "I will drive to the grocery store.", + "bliss": "future:I drive to grocery store. " + }, + { + "original": "She will sing a song at the talent show.", + "bliss": "future:She sing song at talent show. " + }, + { + "original": "They will fly to Europe for vacation.", + "bliss": "future:They fly to Europe for vacation. " + }, + { + "original": "He will cook dinner on the grill.", + "bliss": "future:He cook dinner on grill. " + }, + { + "original": "We will watch fireworks on the Fourth of July.", + "bliss": "future:We watch fireworks on of Fourth July. " + }, + { + "original": "The baby will sleep through the night.", + "bliss": "future:baby sleep through night. " + }, + { + "original": "I will complete the project ahead of schedule.", + "bliss": "future:I complete ahead project of schedule. " + }, + { + "original": "She will knit a sweater for her friend.", + "bliss": "future:She knit sweater for her friend. " + }, + { + "original": "They will take a road trip across the country.", + "bliss": "future:They take road across trip country. " + }, + { + "original": "He will climb to the top of the mountain.", + "bliss": "future:He climb to of top mountain. " + }, + { + "original": "We will bake cookies for the bake sale.", + "bliss": "future:We bake cookies for bake sale. " + }, + { + "original": "The rain will pour down all afternoon.", + "bliss": "future:rain pour down afternoon. " + }, + { + "original": "I will write a letter to my friend.", + "bliss": "future:I write letter to my friend. " + }, + { + "original": "She will sew a dress for the school play.", + "bliss": "future:She sew for dress school play. " + }, + { + "original": "They will plant flowers in the garden.", + "bliss": "future:They plant flowers in garden. " + }, + { + "original": "He will visit his old high school.", + "bliss": "future:He visit his school old high. " + }, + { + "original": "We will cook dinner together as a family.", + "bliss": "future:We cook together dinner as family. " + }, + { + "original": "The birds will chirp in the trees.", + "bliss": "future:birds chirp in trees. " + }, + { + "original": "I will ride my bike to school.", + "bliss": "future:I ride my bike to school. " + }, + { + "original": "She will buy a new car.", + "bliss": "future:She buy car new. " + }, + { + "original": "They will play board games by the fireplace.", + "bliss": "future:They play board games by fireplace. " + }, + { + "original": "He will receive a gift from his parents.", + "bliss": "future:He receive from gift his parents. " + }, + { + "original": "We will attend a concert.", + "bliss": "future:We attend concert. " + }, + { + "original": "The wind will blow fiercely.", + "bliss": "future:wind blow fiercely. " + }, + { + "original": "I will clean my room tomorrow.", + "bliss": "future:I clean my room tomorrow. " + }, + { + "original": "She will confidently walk to the nearby store tomorrow morning.", + "bliss": "future:She walk confidently to store nearby tomorrow morning. " + }, + { + "original": "They will enthusiastically play soccer after energetic school days.", + "bliss": "future:They play enthusiastically soccer after school days energetic. " + }, + { + "original": "He will studiously study for his important exam all night tomorrow.", + "bliss": "future:He study studiously for his exam important night tomorrow. " + }, + { + "original": "We will regularly visit our loving grandparents every cozy weekend.", + "bliss": "future:We visit regularly our love grandparents weekend cozy. " + }, + { + "original": "The playful cat will mischievously chase the startled mouse around the cozy house.", + "bliss": "future:cat playful chase mischievously mouse startled around house cozy. " + }, + { + "original": "I will efficiently finish my challenging homework before delicious dinner tomorrow.", + "bliss": "future:I finish efficiently my homework challenging before dinner delicious tomorrow. " + }, + { + "original": "She will skillfully cook a delectable dinner for her grateful family tomorrow evening.", + "bliss": "future:She cook skillfully dinner delectable for her family grateful tomorrow evening. " + }, + { + "original": "They will quietly watch an entertaining movie at the nearby cinema tomorrow night.", + "bliss": "future:They watch quietly movie entertaining at cinema nearby tomorrow night. " + }, + { + "original": "He will passionately run a challenging marathon next month.", + "bliss": "future:He run passionately marathon challenging month next. " + }, + { + "original": "We will happily go swimming at the beautiful beach next sunny summer.", + "bliss": "future:We go happily swim at beach beautiful summer next sunny. " + }, + { + "original": "The radiant sun will peacefully set over the breathtaking horizon tomorrow.", + "bliss": "future:sun radiant set peacefully over horizon breathtaking tomorrow. " + }, + { + "original": "I will leisurely read an engaging book in bed tomorrow night.", + "bliss": "future:I read leisurely engage book in bed tomorrow night. " + }, + { + "original": "She will carefully paint her bedroom walls a soothing bright blue tomorrow.", + "bliss": "future:She paint carefully her bedroom walls soothe tomorrow. " + }, + { + "original": "He will promptly fix the leaky faucet in the gleaming bathroom tomorrow.", + "bliss": "future:He fix promptly faucet leaky in gleam bathroom tomorrow. " + }, + { + "original": "We will eagerly hike up the scenic mountain trail every refreshing morning.", + "bliss": "future:We hike eagerly up mountain trail scenic morning refreshing. " + }, + { + "original": "The cheerful children will happily play in the spacious backyard every sunny afternoon.", + "bliss": "future:children cheerful play happily in backyard spacious afternoon sunny. " + }, + { + "original": "She will gracefully dance at her sister's enchanting wedding next festive occasion.", + "bliss": "future:She dance gracefully at her sister wedding enchanting next occasion festive. " + }, + { + "original": "They will creatively build a majestic sandcastle at the sandy beach every sunny day next summer.", + "bliss": "future:They build creatively sandcastle majestic at beach sandy day sunny summer next. " + }, + { + "original": "He will wake up early every misty morning to embrace the new day.", + "bliss": "future:He wake up morning misty to embrace day new. " + }, + { + "original": "I will patiently take a crowded bus to work every bustling morning next week.", + "bliss": "future:I take patiently bus crowded to work morning bustling week next. " + }, + { + "original": "She will blissfully swim in the refreshing pool for hours every hot afternoon next summer.", + "bliss": "future:She swim blissfully in for pool refreshing hours afternoon hot summer next. " + }, + { + "original": "They will curiously visit the fascinating museum downtown every educational opportunity.", + "bliss": "future:They visit curiously museum downtown fascinating opportunity educational. " + }, + { + "original": "We will bravely explore the ancient, mysterious ruins every adventurous weekend next month.", + "bliss": "future:We explore bravely ruins ancient mysterious weekend adventurous month next. " + }, + { + "original": "The vigilant dog will fiercely bark at the unsuspecting mailman every busy morning.", + "bliss": "future:dog vigilant bark fiercely at mailman unsuspecting morning busy. " + }, + { + "original": "I will cautiously drive to the bustling grocery store every busy weekend next month.", + "bliss": "future:I drive cautiously to grocery store bustling weekend busy month next. " + }, + { + "original": "She will melodiously sing a beautiful song at the talent show every exciting event next month.", + "bliss": "future:She sing melodiously song beautiful at talent show event exciting month next. " + }, + { + "original": "He will skillfully cook a savory dinner on the sizzling grill every delicious evening.", + "bliss": "future:He cook skillfully dinner savory on grill sizzling evening delicious. " + }, + { + "original": "We will joyfully watch spectacular fireworks on the Fourth of July every celebratory year.", + "bliss": "future:We watch joyfully fireworks spectacular on of Fourth July year celebratory. " + }, + { + "original": "The peaceful baby will quietly sleep through the calm night every cozy evening next month.", + "bliss": "future:baby peaceful sleep quietly through night calm evening cozy month next. " + }, + { + "original": "I will efficiently complete the challenging project ahead of schedule every productive day next week.", + "bliss": "future:I complete efficiently ahead project challenging of schedule day productive week next. " + }, + { + "original": "She will lovingly knit a warm sweater for her grateful friend every chilly season.", + "bliss": "future:She knit lovingly sweater warm for her friend grateful season chilly. " + }, + { + "original": "They will leisurely take a scenic road trip across the vast country every adventurous summer.", + "bliss": "future:They take leisurely road trip scenic across country vast summer adventurous. " + }, + { + "original": "He will courageously climb to the breathtaking top of the tall mountain every adventurous weekend next month.", + "bliss": "future:He climb courageously to of top breathtaking tall mountain tall weekend adventurous month next. " + }, + { + "original": "We will enthusiastically bake delicious cookies for the successful bake sale every festive occasion next month.", + "bliss": "future:We bake enthusiastically cookies delicious for bake sale successful occasion festive month next. " + }, + { + "original": "The relentless rain will pour down heavily all afternoon every gloomy day next week.", + "bliss": "future:rain relentless pour heavily down afternoon day gloomy week next. " + }, + { + "original": "I will quickly write a heartfelt letter to my distant friend every peaceful afternoon next week.", + "bliss": "future:I write quickly letter heartfelt to my friend distant afternoon peaceful week next. " + }, + { + "original": "She will carefully sew a beautiful dress for the exciting school play every exciting occasion next month.", + "bliss": "future:She sew carefully for dress beautiful exciting exciting school play exciting exciting occasion exciting month next. " + }, + { + "original": "They will lovingly plant colorful flowers in the flourishing garden every bright spring next year.", + "bliss": "future:They plant lovingly flowers colorful in flourish garden spring bright year next. " + }, + { + "original": "He will nostalgically visit his old high school for the reunion every memorable year next month.", + "bliss": "future:He visit nostalgically his school old high for reunion year memorable month next. " + }, + { + "original": "We will joyfully cook a hearty dinner together as a happy family every peaceful evening next month.", + "bliss": "future:We cook joyfully together dinner hearty as family happy evening peaceful month next. " + }, + { + "original": "The cheerful birds will happily chirp in the lush, green trees every bright morning next week.", + "bliss": "future:birds cheerful chirp happily in trees lush green morning bright week next. " + }, + { + "original": "I will cheerfully ride my trusty bike to school every sunny morning next month.", + "bliss": "future:I ride cheerfully my bike trusty to school morning sunny month next. " + }, + { + "original": "She will excitedly buy a shiny new car every exciting year.", + "bliss": "future:She buy excitedly car shiny new year exciting. " + }, + { + "original": "They will merrily play engaging board games by the cozy fireplace every cozy evening next month.", + "bliss": "future:They play merrily engage board by games cozy fireplace cozy evening cozy month next. " + }, + { + "original": "He will gratefully receive a thoughtful gift from his loving parents every memorable occasion next month.", + "bliss": "future:He receive gratefully gift thoughtful from his love parents occasion memorable month next. " + }, + { + "original": "We will enthusiastically attend an entertaining concert every exciting weekend next month.", + "bliss": "future:We attend enthusiastically concert entertaining weekend exciting month next. " + }, + { + "original": "The strong wind will fiercely blow through the dark, stormy night every restless evening next month.", + "bliss": "future:wind strong blow fiercely through night dark stormy evening restless month next. " + }, + { + "original": "I will diligently clean my cluttered room every peaceful afternoon next week.", + "bliss": "future:I clean diligently my room cluttered afternoon peaceful week next. " + }, + { + "original": "She walked to school every morning.", + "bliss": "past:She walk to school morning. " + }, + { + "original": "They played soccer in the park yesterday.", + "bliss": "past:They play soccer in park yesterday. " + }, + { + "original": "He visited his grandparents last weekend.", + "bliss": "past:He visit his grandparents weekend last. " + }, + { + "original": "We cooked dinner together last night.", + "bliss": "past:We cook together dinner night last. " + }, + { + "original": "I finished reading the book yesterday.", + "bliss": "past:I finish read book yesterday. " + }, + { + "original": "The sun set beautifully last evening.", + "bliss": "past:sun set beautifully evening last. " + }, + { + "original": "They traveled to Europe last summer.", + "bliss": "past:They travel to Europe summer last. " + }, + { + "original": "She danced at the party last Friday.", + "bliss": "past:She dance at party Friday last. " + }, + { + "original": "He studied hard for his exams last month.", + "bliss": "past:He study hard for his exams month last. " + }, + { + "original": "We visited the museum last week.", + "bliss": "past:We visit museum week last. " + }, + { + "original": "She ate breakfast early yesterday.", + "bliss": "past:She eat breakfast yesterday early. " + }, + { + "original": "They went to the beach last summer.", + "bliss": "past:They go to beach summer last. " + }, + { + "original": "He finished his homework before dinner.", + "bliss": "past:He finish his homework before dinner. " + }, + { + "original": "We watched a movie together last night.", + "bliss": "past:We watch together movie night last. " + }, + { + "original": "I visited my friend yesterday.", + "bliss": "past:I visit my friend yesterday. " + }, + { + "original": "The dog chased the cat around the yard.", + "bliss": "past:dog chase cat around yard. " + }, + { + "original": "They played soccer after school.", + "bliss": "past:They play soccer after school. " + }, + { + "original": "She sang a beautiful song at the concert.", + "bliss": "past:She sing song beautiful at concert. " + }, + { + "original": "He cooked dinner for his family.", + "bliss": "past:He cook dinner for his family. " + }, + { + "original": "We traveled to Europe last year.", + "bliss": "past:We travel to Europe year last. " + }, + { + "original": "She studied at the library yesterday.", + "bliss": "past:She study at library yesterday. " + }, + { + "original": "They went for a hike last weekend.", + "bliss": "past:They go for hike weekend last. " + }, + { + "original": "He finished his dinner early last night.", + "bliss": "past:He finish his dinner night last. " + }, + { + "original": "We visited the zoo last month.", + "bliss": "past:We visit zoo month last. " + }, + { + "original": "I cooked spaghetti for dinner yesterday.", + "bliss": "past:I cook spaghetti for dinner yesterday. " + }, + { + "original": "The movie ended late last evening.", + "bliss": "past:movie end evening last. " + }, + { + "original": "They traveled to Japan last summer.", + "bliss": "past:They travel to Japan summer last. " + }, + { + "original": "She danced at her sister's wedding last Friday.", + "bliss": "past:She dance at her sister wedding Friday last. " + }, + { + "original": "He played the piano at the concert last week.", + "bliss": "past:He play piano at concert week last. " + }, + { + "original": "We completed the puzzle yesterday.", + "bliss": "past:We complete puzzle yesterday. " + }, + { + "original": "She woke up late yesterday morning.", + "bliss": "past:She wake up yesterday morning late. " + }, + { + "original": "They played basketball at the park yesterday.", + "bliss": "past:They play basketball at park yesterday. " + }, + { + "original": "He visited his aunt yesterday afternoon.", + "bliss": "past:He visit his aunt yesterday afternoon. " + }, + { + "original": "We cleaned the house yesterday.", + "bliss": "past:We clean house yesterday. " + }, + { + "original": "I watched a movie with my friends yesterday evening.", + "bliss": "past:I watch movie with my friends yesterday evening. " + }, + { + "original": "The sun set beautifully yesterday.", + "bliss": "past:sun set beautifully yesterday. " + }, + { + "original": "They had a barbecue in their backyard yesterday.", + "bliss": "past:They have barbecue in their backyard yesterday. " + }, + { + "original": "She finished her homework yesterday.", + "bliss": "past:She finish her homework yesterday. " + }, + { + "original": "He went for a run yesterday.", + "bliss": "past:He go for run yesterday. " + }, + { + "original": "We attended a party yesterday night.", + "bliss": "past:We attend party yesterday night. " + }, + { + "original": "She walked to school yesterday.", + "bliss": "past:She walk to school yesterday. " + }, + { + "original": "He ate breakfast early this morning.", + "bliss": "past:He eat breakfast morning. " + }, + { + "original": "They watched a movie last night.", + "bliss": "past:They watch movie night last. " + }, + { + "original": "I visited my grandparents last weekend.", + "bliss": "past:I visit my grandparents weekend last. " + }, + { + "original": "We played soccer after school yesterday.", + "bliss": "past:We play soccer after school yesterday. " + }, + { + "original": "The sun shone brightly yesterday.", + "bliss": "past:sun shine brightly yesterday. " + }, + { + "original": "He finished his homework before dinner.", + "bliss": "past:He finish his homework before dinner. " + }, + { + "original": "They went swimming at the beach last summer.", + "bliss": "past:They go swim at beach summer last. " + }, + { + "original": "She cooked dinner for her family yesterday.", + "bliss": "past:She cook dinner for her family yesterday. " + }, + { + "original": "The cat slept all day yesterday.", + "bliss": "past:cat sleep day yesterday. " + }, + { + "original": "He ran five miles yesterday morning.", + "bliss": "past:He run miles yesterday morning. " + }, + { + "original": "They cleaned the house last weekend.", + "bliss": "past:They clean house weekend last. " + }, + { + "original": "I read a book before going to bed last night.", + "bliss": "present:I read book before go to bed night last. " + }, + { + "original": "We visited the museum last month.", + "bliss": "past:We visit museum month last. " + }, + { + "original": "She bought a new dress last week.", + "bliss": "past:She buy dress new week last. " + }, + { + "original": "He rode his bike to work yesterday.", + "bliss": "past:He ride his bike to work yesterday. " + }, + { + "original": "They danced at the party last night.", + "bliss": "past:They dance at party night last. " + }, + { + "original": "I graduated from college last year.", + "bliss": "past:I graduate from college year last. " + }, + { + "original": "We went on a road trip last summer.", + "bliss": "past:We go on road trip summer last. " + }, + { + "original": "She painted a beautiful picture last night.", + "bliss": "past:She paint picture beautiful night last. " + }, + { + "original": "He fixed the broken window last weekend.", + "bliss": "past:He fix window broken weekend last. " + }, + { + "original": "They sang songs around the campfire last night.", + "bliss": "past:They sing around songs campfire night last. " + }, + { + "original": "I woke up early this morning.", + "bliss": "past:I wake up morning. " + }, + { + "original": "We played board games all night yesterday.", + "bliss": "past:We play board games night yesterday. " + }, + { + "original": "She baked cookies for her friends yesterday.", + "bliss": "past:She bake cookies for her friends yesterday. " + }, + { + "original": "He traveled to Europe last summer.", + "bliss": "past:He travel to Europe summer last. " + }, + { + "original": "They attended a concert last night.", + "bliss": "past:They attend concert night last. " + }, + { + "original": "I finished reading the book last week.", + "bliss": "past:I finish read book week last. " + }, + { + "original": "We went fishing at the lake last weekend.", + "bliss": "past:We go fish at lake weekend last. " + }, + { + "original": "She practiced the piano for hours yesterday.", + "bliss": "past:She practice piano for hours yesterday. " + }, + { + "original": "He cleaned his room yesterday.", + "bliss": "past:He clean his room yesterday. " + }, + { + "original": "They planted flowers in the garden last spring.", + "bliss": "past:They plant flowers in garden spring last. " + }, + { + "original": "I watched a documentary on TV last night.", + "bliss": "past:I watch on documentary TV night last. " + }, + { + "original": "We had a barbecue in the backyard last weekend.", + "bliss": "past:We have barbecue in backyard weekend last. " + }, + { + "original": "She studied for her exams last week.", + "bliss": "past:She study for her exams week last. " + }, + { + "original": "He fixed the leaky faucet last weekend.", + "bliss": "past:He fix faucet leaky weekend last. " + }, + { + "original": "They walked in the park yesterday.", + "bliss": "past:They walk in park yesterday. " + }, + { + "original": "I played basketball with my friends last night.", + "bliss": "past:I play basketball with my friends night last. " + }, + { + "original": "We visited the zoo last weekend.", + "bliss": "past:We visit zoo weekend last. " + }, + { + "original": "She wrote a letter to her pen pal yesterday.", + "bliss": "past:She write to letter her pen pal yesterday. " + }, + { + "original": "He mowed the lawn yesterday.", + "bliss": "past:He mow lawn yesterday. " + }, + { + "original": "They went shopping at the mall last weekend.", + "bliss": "past:They go shop at mall weekend last. " + }, + { + "original": "I went for a run yesterday morning.", + "bliss": "past:I go for run yesterday morning. " + }, + { + "original": "We watched the sunset at the beach last night.", + "bliss": "past:We watch sunset at beach night last. " + }, + { + "original": "She finished knitting a sweater last week.", + "bliss": "past:She finish knit sweater week last. " + }, + { + "original": "He cooked dinner for his family yesterday.", + "bliss": "past:He cook dinner for his family yesterday. " + }, + { + "original": "They hiked in the mountains last weekend.", + "bliss": "past:They hike in mountains weekend last. " + }, + { + "original": "I cleaned out the garage yesterday.", + "bliss": "past:I clean out garage yesterday. " + }, + { + "original": "We had a picnic in the park last weekend.", + "bliss": "past:We have in picnic park weekend last. " + }, + { + "original": "She painted her nails yesterday.", + "bliss": "past:She paint her nails yesterday. " + }, + { + "original": "He read a book in the garden yesterday.", + "bliss": "past:He read book in garden yesterday. " + }, + { + "original": "They played soccer in the park yesterday.", + "bliss": "past:They play soccer in park yesterday. " + }, + { + "original": "I attended a party last night.", + "bliss": "past:I attend party night last. " + }, + { + "original": "We rode bikes around the neighborhood last weekend.", + "bliss": "past:We ride around bikes neighborhood weekend last. " + }, + { + "original": "She watched a movie with her friends last night.", + "bliss": "past:She watch movie with her friends night last. " + }, + { + "original": "He went for a swim at the pool yesterday.", + "bliss": "past:He go for at swim pool yesterday. " + }, + { + "original": "They went for a hike in the forest yesterday.", + "bliss": "past:They go for in hike forest yesterday. " + }, + { + "original": "I visited my aunt last weekend.", + "bliss": "past:I visit my aunt weekend last. " + }, + { + "original": "We played video games all day yesterday.", + "bliss": "past:We play video games day yesterday. " + }, + { + "original": "She cooked a delicious meal yesterday.", + "bliss": "past:She cook meal delicious yesterday. " + }, + { + "original": "He rode his skateboard to the park yesterday.", + "bliss": "past:He ride his skateboard to park yesterday. " + }, + { + "original": "They had a barbecue in the backyard yesterday.", + "bliss": "past:They have barbecue in backyard yesterday. " + }, + { + "original": "I went camping with my family last weekend.", + "bliss": "past:I go camp with my family weekend last. " + }, + { + "original": "We went to the beach last weekend.", + "bliss": "past:We go to beach weekend last. " + }, + { + "original": "She practiced yoga in the morning yesterday.", + "bliss": "past:She practice yoga in morning yesterday. " + }, + { + "original": "He listened to music in his room yesterday.", + "bliss": "past:He listen to in music his room yesterday. " + }, + { + "original": "They had a sleepover at their friend's house last weekend.", + "bliss": "past:They have sleepover at their friend house weekend last. " + }, + { + "original": "I went for a bike ride yesterday.", + "bliss": "past:I go for bike ride yesterday. " + }, + { + "original": "We went for a walk in the park yesterday.", + "bliss": "past:We go for in walk park yesterday. " + }, + { + "original": "She took a nap in the afternoon yesterday.", + "bliss": "past:She take nap in afternoon yesterday. " + }, + { + "original": "He played guitar for his friends yesterday.", + "bliss": "past:He play guitar for his friends yesterday. " + }, + { + "original": "They went for a drive in the countryside yesterday.", + "bliss": "past:They go for in drive countryside yesterday. " + }, + { + "original": "I visited the library last week.", + "bliss": "past:I visit library week last. " + }, + { + "original": "We went to a party last night.", + "bliss": "past:We go to party night last. " + }, + { + "original": "She swam in the pool yesterday.", + "bliss": "past:She swim in pool yesterday. " + }, + { + "original": "He cooked breakfast for his family yesterday.", + "bliss": "past:He cook breakfast for his family yesterday. " + }, + { + "original": "They watched a movie at home yesterday.", + "bliss": "past:They watch movie at home yesterday. " + }, + { + "original": "I played video games with my brother yesterday.", + "bliss": "past:I play video games with my brother yesterday. " + }, + { + "original": "We went to the amusement park last weekend.", + "bliss": "past:We go to amusement park weekend last. " + }, + { + "original": "She went shopping with her friends yesterday.", + "bliss": "past:She go shop with her friends yesterday. " + }, + { + "original": "He rode his bike to school yesterday.", + "bliss": "past:He ride his bike to school yesterday. " + }, + { + "original": "They visited their grandparents last weekend.", + "bliss": "past:They visit their grandparents weekend last. " + }, + { + "original": "I cleaned my room yesterday.", + "bliss": "past:I clean my room yesterday. " + }, + { + "original": "We had a picnic in the backyard last weekend.", + "bliss": "past:We have in picnic backyard weekend last. " + }, + { + "original": "She read a book in the garden yesterday.", + "bliss": "past:She read book in garden yesterday. " + }, + { + "original": "He played soccer with his friends yesterday.", + "bliss": "past:He play soccer with his friends yesterday. " + }, + { + "original": "They had a barbecue at the beach yesterday.", + "bliss": "past:They have barbecue at beach yesterday. " + }, + { + "original": "I went for a swim in the lake yesterday.", + "bliss": "past:I go for in swim lake yesterday. " + }, + { + "original": "We watched the sunset at the park last weekend.", + "bliss": "past:We watch sunset at park weekend last. " + }, + { + "original": "She went hiking with her friends yesterday.", + "bliss": "past:She go hike with her friends yesterday. " + }, + { + "original": "He rode his skateboard at the skate park yesterday.", + "bliss": "past:He ride his skateboard at skate park yesterday. " + }, + { + "original": "They had a picnic in the forest yesterday.", + "bliss": "past:They have in picnic forest yesterday. " + }, + { + "original": "I visited the museum last weekend.", + "bliss": "past:I visit museum weekend last. " + }, + { + "original": "We went to a concert last night.", + "bliss": "past:We go to concert night last. " + }, + { + "original": "She played tennis with her friends yesterday.", + "bliss": "past:She play tennis with her friends yesterday. " + }, + { + "original": "He went fishing with his dad yesterday.", + "bliss": "past:He go fish with his dad yesterday. " + }, + { + "original": "They had a barbecue in the park yesterday.", + "bliss": "past:They have in barbecue park yesterday. " + }, + { + "original": "I went for a jog in the morning yesterday.", + "bliss": "past:I go for in jog morning yesterday. " + }, + { + "original": "We had a bonfire at the beach last weekend.", + "bliss": "past:We have at bonfire beach weekend last. " + }, + { + "original": "She went for a bike ride in the countryside yesterday.", + "bliss": "past:She go for bike in ride countryside yesterday. " + }, + { + "original": "She walked to the store yesterday.", + "bliss": "past:She walk to store yesterday. " + }, + { + "original": "They played soccer after school.", + "bliss": "past:They play soccer after school. " + }, + { + "original": "He studied for his exam all night.", + "bliss": "past:He study for his exam night. " + }, + { + "original": "We visited Paris last summer.", + "bliss": "past:We visit Paris summer last. " + }, + { + "original": "The cat chased the mouse around the house.", + "bliss": "past:cat chase mouse around house. " + }, + { + "original": "I finished my homework before dinner.", + "bliss": "past:I finish my homework before dinner. " + }, + { + "original": "She cooked dinner for her family.", + "bliss": "past:She cook dinner for her family. " + }, + { + "original": "They watched a movie at the cinema.", + "bliss": "past:They watch movie at cinema. " + }, + { + "original": "He ran a marathon last month.", + "bliss": "past:He run marathon month last. " + }, + { + "original": "We went camping in the mountains.", + "bliss": "past:We go in camping mountains. " + }, + { + "original": "The sun set over the horizon.", + "bliss": "past:sun set over horizon. " + }, + { + "original": "I read a book in bed last night.", + "bliss": "present:I read book in bed night last. " + }, + { + "original": "She painted her bedroom walls blue.", + "bliss": "past:She paint her bedroom walls. " + }, + { + "original": "They visited their grandparents over the holidays.", + "bliss": "past:They visit their grandparents over holidays. " + }, + { + "original": "He fixed the leaky faucet in the bathroom.", + "bliss": "past:He fix faucet leaky in bathroom. " + }, + { + "original": "We rode bikes through the park.", + "bliss": "past:We ride bikes through park. " + }, + { + "original": "The children played in the backyard.", + "bliss": "past:children play in backyard. " + }, + { + "original": "She danced at her sister's wedding.", + "bliss": "past:She dance at her sister wedding. " + }, + { + "original": "They built a sandcastle at the beach.", + "bliss": "past:They build sandcastle at beach. " + }, + { + "original": "He woke up early this morning.", + "bliss": "past:He wake up morning. " + }, + { + "original": "I ate breakfast before leaving for work.", + "bliss": "past:I eat breakfast before leave for work. " + }, + { + "original": "She swam in the pool for hours.", + "bliss": "past:She swim in pool for hours. " + }, + { + "original": "They visited the museum downtown.", + "bliss": "past:They visit museum downtown. " + }, + { + "original": "We hiked up the mountain trail.", + "bliss": "past:We hike up mountain trail. " + }, + { + "original": "The dog barked at the mailman.", + "bliss": "past:dog bark at mailman. " + }, + { + "original": "I drove to the grocery store.", + "bliss": "past:I drive to grocery store. " + }, + { + "original": "She sang a song at the talent show.", + "bliss": "past:She sing song at talent show. " + }, + { + "original": "They flew to Europe for vacation.", + "bliss": "past:They fly to Europe for vacation. " + }, + { + "original": "He cooked dinner on the grill.", + "bliss": "past:He cook dinner on grill. " + }, + { + "original": "We watched fireworks on the Fourth of July.", + "bliss": "past:We watch of fireworks of Fourth July. " + }, + { + "original": "The baby slept through the night.", + "bliss": "past:baby sleep through night. " + }, + { + "original": "I completed the project ahead of schedule.", + "bliss": "past:I complete ahead project of schedule. " + }, + { + "original": "She knitted a sweater for her friend.", + "bliss": "past:She knit sweater for her friend. " + }, + { + "original": "They took a road trip across the country.", + "bliss": "past:They take road across trip country. " + }, + { + "original": "He climbed to the top of the mountain.", + "bliss": "past:He climb to of top mountain. " + }, + { + "original": "We explored the ancient ruins.", + "bliss": "past:We explore ruins ancient. " + }, + { + "original": "The train arrived at the station.", + "bliss": "past:train arrive at station. " + }, + { + "original": "I finished reading the novel.", + "bliss": "past:I finish read novel. " + }, + { + "original": "She wrote a letter to her pen pal.", + "bliss": "past:She write letter to her pen pal. " + }, + { + "original": "They visited the zoo on Saturday.", + "bliss": "past:They visit zoo on Saturday. " + }, + { + "original": "He fixed the broken window.", + "bliss": "past:He fix window broken. " + }, + { + "original": "We played board games by the fireplace.", + "bliss": "past:We play board games by fireplace. " + }, + { + "original": "The birds chirped in the trees.", + "bliss": "past:birds chirp in trees. " + }, + { + "original": "I rode my bike to school.", + "bliss": "past:I ride my bike to school. " + }, + { + "original": "She sewed a dress for the school play.", + "bliss": "past:She sew for dress school play. " + }, + { + "original": "They planted flowers in the garden.", + "bliss": "past:They plant flowers in garden. " + }, + { + "original": "He visited his old high school.", + "bliss": "past:He visit his school old high. " + }, + { + "original": "We cooked dinner together as a family.", + "bliss": "past:We cook together dinner as family. " + }, + { + "original": "The rain poured down all afternoon.", + "bliss": "past:rain pour down afternoon. " + }, + { + "original": "I baked cookies for the bake sale.", + "bliss": "past:I bake cookies for bake sale. " + }, + { + "original": "She ate an apple yesterday.", + "bliss": "past:She eat apple yesterday. " + }, + { + "original": "They walked to the park last night.", + "bliss": "past:They walk to park night last. " + }, + { + "original": "He finished his homework before dinner.", + "bliss": "past:He finish his homework before dinner. " + }, + { + "original": "We visited our grandparents last weekend.", + "bliss": "past:We visit our grandparents weekend last. " + }, + { + "original": "The cat chased the mouse around the house.", + "bliss": "past:cat chase mouse around house. " + }, + { + "original": "I played basketball with my friends yesterday.", + "bliss": "past:I play basketball with my friends yesterday. " + }, + { + "original": "She cooked dinner for her family.", + "bliss": "past:She cook dinner for her family. " + }, + { + "original": "They watched a movie at the cinema.", + "bliss": "past:They watch movie at cinema. " + }, + { + "original": "He ran a marathon last month.", + "bliss": "past:He run marathon month last. " + }, + { + "original": "We went swimming at the beach last summer.", + "bliss": "past:We go swim at beach summer last. " + }, + { + "original": "The sun set over the horizon.", + "bliss": "past:sun set over horizon. " + }, + { + "original": "I read a book in bed last night.", + "bliss": "present:I read book in bed night last. " + }, + { + "original": "She painted her bedroom walls blue.", + "bliss": "past:She paint her bedroom walls. " + }, + { + "original": "They traveled to Europe for vacation.", + "bliss": "past:They travel to Europe for vacation. " + }, + { + "original": "He fixed the leaky faucet in the bathroom.", + "bliss": "past:He fix faucet leaky in bathroom. " + }, + { + "original": "We hiked up the mountain trail.", + "bliss": "past:We hike up mountain trail. " + }, + { + "original": "The children played in the backyard.", + "bliss": "past:children play in backyard. " + }, + { + "original": "She danced at her sister's wedding.", + "bliss": "past:She dance at her sister wedding. " + }, + { + "original": "They built a sandcastle at the beach.", + "bliss": "past:They build sandcastle at beach. " + }, + { + "original": "He woke up early this morning.", + "bliss": "past:He wake up morning. " + }, + { + "original": "I took a bus to work yesterday.", + "bliss": "past:I take bus to work yesterday. " + }, + { + "original": "She swam in the pool for hours.", + "bliss": "past:She swim in pool for hours. " + }, + { + "original": "They visited the museum downtown.", + "bliss": "past:They visit museum downtown. " + }, + { + "original": "We explored the ancient ruins.", + "bliss": "past:We explore ruins ancient. " + }, + { + "original": "The dog barked at the mailman.", + "bliss": "past:dog bark at mailman. " + }, + { + "original": "I drove to the grocery store.", + "bliss": "past:I drive to grocery store. " + }, + { + "original": "She sang a song at the talent show.", + "bliss": "past:She sing song at talent show. " + }, + { + "original": "They flew to Europe for vacation.", + "bliss": "past:They fly to Europe for vacation. " + }, + { + "original": "He cooked dinner on the grill.", + "bliss": "past:He cook dinner on grill. " + }, + { + "original": "We watched fireworks on the Fourth of July.", + "bliss": "past:We watch of fireworks of Fourth July. " + }, + { + "original": "The baby slept through the night.", + "bliss": "past:baby sleep through night. " + }, + { + "original": "I completed the project ahead of schedule.", + "bliss": "past:I complete ahead project of schedule. " + }, + { + "original": "She knitted a sweater for her friend.", + "bliss": "past:She knit sweater for her friend. " + }, + { + "original": "They took a road trip across the country.", + "bliss": "past:They take road across trip country. " + }, + { + "original": "He climbed to the top of the mountain.", + "bliss": "past:He climb to of top mountain. " + }, + { + "original": "We baked cookies for the bake sale.", + "bliss": "past:We bake cookies for bake sale. " + }, + { + "original": "The rain poured down all afternoon.", + "bliss": "past:rain pour down afternoon. " + }, + { + "original": "I wrote a letter to my friend yesterday.", + "bliss": "past:I write to letter my friend yesterday. " + }, + { + "original": "She sewed a dress for the school play.", + "bliss": "past:She sew for dress school play. " + }, + { + "original": "They planted flowers in the garden.", + "bliss": "past:They plant flowers in garden. " + }, + { + "original": "He visited his old high school.", + "bliss": "past:He visit his school old high. " + }, + { + "original": "We cooked dinner together as a family.", + "bliss": "past:We cook together dinner as family. " + }, + { + "original": "The birds chirped in the trees.", + "bliss": "past:birds chirp in trees. " + }, + { + "original": "I rode my bike to school.", + "bliss": "past:I ride my bike to school. " + }, + { + "original": "She bought a new car last month.", + "bliss": "past:She buy car new month last. " + }, + { + "original": "They played board games by the fireplace.", + "bliss": "past:They play board games by fireplace. " + }, + { + "original": "He received a gift from his parents.", + "bliss": "past:He receive gift from his parents. " + }, + { + "original": "We attended a concert last night.", + "bliss": "past:We attend concert night last. " + }, + { + "original": "The wind blew fiercely last night.", + "bliss": "past:wind blow fiercely night last. " + }, + { + "original": "I cleaned my room yesterday.", + "bliss": "past:I clean my room yesterday. " + }, + { + "original": "She quickly ate a juicy apple yesterday.", + "bliss": "past:She eat quickly apple juicy yesterday. " + }, + { + "original": "They happily walked to the park last night.", + "bliss": "past:They walk happily to park night last. " + }, + { + "original": "He diligently finished his difficult homework before dinner.", + "bliss": "past:He finish diligently his homework difficult before dinner. " + }, + { + "original": "We joyfully visited our beloved grandparents last weekend.", + "bliss": "past:We visit joyfully our grandparents beloved weekend last. " + }, + { + "original": "The agile cat playfully chased the frightened mouse around the house.", + "bliss": "past:cat agile chase playfully mouse frightened around house. " + }, + { + "original": "I enthusiastically played basketball with my friends yesterday.", + "bliss": "past:I play enthusiastically basketball with my friends yesterday. " + }, + { + "original": "She skillfully cooked a delicious dinner for her family.", + "bliss": "past:She cook skillfully dinner delicious for her family. " + }, + { + "original": "They quietly watched a thrilling movie at the cinema.", + "bliss": "past:They watch quietly at movie thrilling cinema. " + }, + { + "original": "He proudly ran a challenging marathon last month.", + "bliss": "past:He run proudly marathon challenging month last. " + }, + { + "original": "We peacefully went swimming at the serene beach last summer.", + "bliss": "past:We go peacefully swim at beach serene summer last. " + }, + { + "original": "The radiant sun peacefully set over the picturesque horizon.", + "bliss": "past:sun radiant set peacefully over horizon picturesque. " + }, + { + "original": "I leisurely read an intriguing book in bed last night.", + "bliss": "past:I read leisurely book intriguing in bed night last. " + }, + { + "original": "She carefully painted her bedroom walls bright blue.", + "bliss": "past:She paint carefully her bedroom walls. " + }, + { + "original": "They excitedly traveled to beautiful Europe for vacation.", + "bliss": "past:They travel excitedly to Europe beautiful for vacation. " + }, + { + "original": "He efficiently fixed the leaky faucet in the bathroom.", + "bliss": "past:He fix efficiently faucet leaky in bathroom. " + }, + { + "original": "We eagerly hiked up the scenic mountain trail.", + "bliss": "past:We hike eagerly up mountain trail scenic. " + }, + { + "original": "The cheerful children happily played in the spacious backyard.", + "bliss": "past:children cheerful play happily in backyard spacious. " + }, + { + "original": "She gracefully danced at her sister's enchanting wedding.", + "bliss": "past:She dance gracefully at her sister wedding enchanting. " + }, + { + "original": "They creatively built a majestic sandcastle at the sandy beach.", + "bliss": "past:They build creatively sandcastle majestic at beach sandy. " + }, + { + "original": "He woke up early this misty morning.", + "bliss": "past:He wake up morning misty. " + }, + { + "original": "I patiently took a crowded bus to work yesterday.", + "bliss": "past:I take patiently bus crowded to work yesterday. " + }, + { + "original": "She blissfully swam in the refreshing pool for hours.", + "bliss": "present:She swam blissfully in for pool refreshing hours. " + }, + { + "original": "They curiously visited the fascinating museum downtown.", + "bliss": "past:They visit curiously museum downtown fascinating. " + }, + { + "original": "We bravely explored the ancient, mysterious ruins.", + "bliss": "past:We explore bravely ruins ancient mysterious. " + }, + { + "original": "The vigilant dog fiercely barked at the unsuspecting mailman.", + "bliss": "past:dog vigilant bark fiercely at mailman unsuspecting. " + }, + { + "original": "I cautiously drove to the busy grocery store.", + "bliss": "past:I drive cautiously to grocery store busy. " + }, + { + "original": "She melodiously sang a beautiful song at the talent show.", + "bliss": "past:She sing melodiously song beautiful at talent show. " + }, + { + "original": "They eagerly flew to romantic Europe for vacation.", + "bliss": "past:They fly eagerly to Europe romantic for vacation. " + }, + { + "original": "He skillfully cooked a savory dinner on the sizzling grill.", + "bliss": "past:He cook skillfully dinner savory on grill sizzling. " + }, + { + "original": "We joyfully watched spectacular fireworks on the Fourth of July.", + "bliss": "past:We watch joyfully of fireworks spectacular of Fourth July. " + }, + { + "original": "The peaceful baby quietly slept through the calm night.", + "bliss": "past:baby peaceful sleep quietly through night calm. " + }, + { + "original": "I efficiently completed the challenging project ahead of schedule.", + "bliss": "past:I complete efficiently ahead project challenging of schedule. " + }, + { + "original": "She lovingly knitted a warm sweater for her grateful friend.", + "bliss": "past:She knit lovingly sweater warm for her friend grateful. " + }, + { + "original": "They leisurely took a scenic road trip across the vast country.", + "bliss": "past:They take leisurely road trip scenic across country vast. " + }, + { + "original": "He courageously climbed to the breathtaking top of the tall mountain.", + "bliss": "past:He climb courageously to of top breathtaking tall mountain tall. " + }, + { + "original": "We enthusiastically baked delicious cookies for the successful bake sale.", + "bliss": "past:We bake enthusiastically cookies delicious for bake sale successful. " + }, + { + "original": "The relentless rain poured down heavily all afternoon.", + "bliss": "past:rain relentless pour heavily down afternoon. " + }, + { + "original": "I quickly wrote a heartfelt letter to my distant friend yesterday.", + "bliss": "past:I write quickly letter heartfelt to my friend distant yesterday. " + }, + { + "original": "She carefully sewed a beautiful dress for the exciting school play.", + "bliss": "past:She sew carefully for dress beautiful exciting school play exciting. " + }, + { + "original": "They lovingly planted colorful flowers in the flourishing garden.", + "bliss": "past:They plant lovingly flowers colorful in flourish garden. " + }, + { + "original": "He nostalgically visited his old high school for the reunion.", + "bliss": "past:He visit nostalgically his for school old high reunion. " + }, + { + "original": "We joyfully cooked a hearty dinner together as a happy family.", + "bliss": "past:We cook joyfully together dinner hearty as family happy. " + }, + { + "original": "The cheerful birds happily chirped in the lush, green trees.", + "bliss": "past:birds cheerful chirp happily in trees lush green. " + }, + { + "original": "I cheerfully rode my trusty bike to school every sunny morning.", + "bliss": "past:I ride cheerfully my bike trusty to school morning sunny. " + }, + { + "original": "She excitedly bought a shiny new car last month.", + "bliss": "past:She buy excitedly car shiny new month last. " + }, + { + "original": "They merrily played engaging board games by the cozy fireplace.", + "bliss": "past:They play merrily engage board by games cozy fireplace cozy. " + }, + { + "original": "He gratefully received a thoughtful gift from his loving parents.", + "bliss": "past:He receive gratefully gift thoughtful from his love parents. " + }, + { + "original": "We enthusiastically attended an entertaining concert last night.", + "bliss": "past:We attend enthusiastically concert entertaining night last. " + }, + { + "original": "The strong wind fiercely blew through the dark, stormy night.", + "bliss": "past:wind strong blow fiercely through night dark stormy. " + }, + { + "original": "I diligently cleaned my cluttered room yesterday afternoon.", + "bliss": "past:I clean diligently my room cluttered yesterday afternoon. " + }, + { + "original": "She walks to school every morning.", + "bliss": "present:She walk to school morning. " + }, + { + "original": "They play soccer in the park today.", + "bliss": "present:They play soccer in park today. " + }, + { + "original": "He visits his grandparents every weekend.", + "bliss": "present:He visit his grandparents weekend. " + }, + { + "original": "We cook dinner together every night.", + "bliss": "present:We cook together dinner night. " + }, + { + "original": "I am reading the book today.", + "bliss": "present progressive:I be read book today. " + }, + { + "original": "The sun is setting beautifully this evening.", + "bliss": "present progressive:sun be set beautifully evening. " + }, + { + "original": "They are traveling to Europe this summer.", + "bliss": "present progressive:They be travel to Europe summer. " + }, + { + "original": "She dances at the party every Friday.", + "bliss": "present:She dance at party Friday. " + }, + { + "original": "He studies hard for his exams this month.", + "bliss": "present:He study for his exams month. " + }, + { + "original": "We visit the museum every week.", + "bliss": "present:We visit museum week. " + }, + { + "original": "She eats breakfast every morning.", + "bliss": "present:She eat breakfast morning. " + }, + { + "original": "They go to the beach every summer.", + "bliss": "present:They go to beach summer. " + }, + { + "original": "He finishes his homework before dinner.", + "bliss": "present:He finish his homework before dinner. " + }, + { + "original": "We watch a movie together every night.", + "bliss": "present:We watch together movie night. " + }, + { + "original": "I visit my friend regularly.", + "bliss": "present:I visit regularly my friend. " + }, + { + "original": "The dog chases the cat around the yard.", + "bliss": "present:dog chase cat around yard. " + }, + { + "original": "They play soccer after school.", + "bliss": "present:They play soccer after school. " + }, + { + "original": "She sings a beautiful song at the concert.", + "bliss": "present:She sing song beautiful at concert. " + }, + { + "original": "He cooks dinner for his family.", + "bliss": "present:He cook for dinner his family. " + }, + { + "original": "We travel to Europe every year.", + "bliss": "present:We travel to Europe year. " + }, + { + "original": "They are playing soccer in the park today.", + "bliss": "present progressive:They be play soccer in park today. " + }, + { + "original": "He visits his grandparents every weekend.", + "bliss": "present:He visit his grandparents weekend. " + }, + { + "original": "We cook dinner together every night.", + "bliss": "present:We cook together dinner night. " + }, + { + "original": "I am reading the book right now.", + "bliss": "present progressive:I be read now book. " + }, + { + "original": "The sun is shining brightly today.", + "bliss": "present progressive:sun be shine brightly today. " + }, + { + "original": "They travel to Europe every summer.", + "bliss": "present:They travel to Europe summer. " + }, + { + "original": "She dances at the party every Friday.", + "bliss": "present:She dance at party Friday. " + }, + { + "original": "He studies hard for his exams every month.", + "bliss": "present:He study for his exams month. " + }, + { + "original": "We visit the museum every week.", + "bliss": "present:We visit museum week. " + }, + { + "original": "She is studying at the library right now.", + "bliss": "present progressive:She be study now at library. " + }, + { + "original": "They are going for a hike this weekend.", + "bliss": "present progressive:They be go for hike weekend. " + }, + { + "original": "He finishes his dinner early every night.", + "bliss": "present:He finish his dinner night. " + }, + { + "original": "We visit the zoo every month.", + "bliss": "present:We visit zoo month. " + }, + { + "original": "I am cooking spaghetti for dinner tonight.", + "bliss": "present progressive:I be cook spaghetti for dinner tonight. " + }, + { + "original": "The movie is playing at the theater right now.", + "bliss": "present progressive:movie be play now at theater. " + }, + { + "original": "They are planning to travel to Japan this summer.", + "bliss": "present progressive:They be plan to travel to Japan summer. " + }, + { + "original": "She dances every Friday evening.", + "bliss": "present:She dance Friday evening. " + }, + { + "original": "He plays the piano at the concert every week.", + "bliss": "present:He play piano at concert week. " + }, + { + "original": "We are solving a puzzle right now.", + "bliss": "present progressive:We be solve now puzzle. " + }, + { + "original": "She wakes up early every morning.", + "bliss": "present:She wake up morning. " + }, + { + "original": "They play basketball at the park every weekend.", + "bliss": "present:They play basketball at park weekend. " + }, + { + "original": "He visits his aunt regularly.", + "bliss": "present:He visit regularly his aunt. " + }, + { + "original": "We clean the house every week.", + "bliss": "present:We clean house week. " + }, + { + "original": "I watch a movie with my friends every Friday night.", + "bliss": "present:I watch movie with my friends Friday night. " + }, + { + "original": "The sun shines brightly every day.", + "bliss": "present:sun shine brightly day. " + }, + { + "original": "They have a barbecue in their backyard every summer.", + "bliss": "present:They have barbecue in their backyard summer. " + }, + { + "original": "She finishes her homework on time every day.", + "bliss": "present:She finish her homework on time day. " + }, + { + "original": "He goes for a run every morning.", + "bliss": "present:He go for run morning. " + }, + { + "original": "We attend parties occasionally.", + "bliss": "present:We attend occasionally parties. " + }, + { + "original": "She walks to school every day.", + "bliss": "present:She walk to school day. " + }, + { + "original": "He eats breakfast early every morning.", + "bliss": "present:He eat breakfast morning. " + }, + { + "original": "They watch a movie every night.", + "bliss": "present:They watch movie night. " + }, + { + "original": "I visit my grandparents every weekend.", + "bliss": "present:I visit my grandparents weekend. " + }, + { + "original": "We play soccer after school every day.", + "bliss": "present:We play soccer after school day. " + }, + { + "original": "The sun shines brightly every day.", + "bliss": "present:sun shine brightly day. " + }, + { + "original": "He finishes his homework before dinner every day.", + "bliss": "present:He finish his homework before dinner day. " + }, + { + "original": "They go swimming at the beach every summer.", + "bliss": "present:They go swim at beach summer. " + }, + { + "original": "She cooks dinner for her family every day.", + "bliss": "present:She cook for dinner her family day. " + }, + { + "original": "The cat sleeps all day every day.", + "bliss": "present:cat sleep day day. " + }, + { + "original": "He runs five miles every morning.", + "bliss": "present:He run miles morning. " + }, + { + "original": "They clean the house every weekend.", + "bliss": "present:They clean house weekend. " + }, + { + "original": "I read a book before going to bed every night.", + "bliss": "present:I read book before go to bed night. " + }, + { + "original": "We visit the museum every month.", + "bliss": "present:We visit museum month. " + }, + { + "original": "She buys a new dress every week.", + "bliss": "present:She buy dress new week. " + }, + { + "original": "He rides his bike to work every day.", + "bliss": "present:He ride his bike to work day. " + }, + { + "original": "They dance at the party every night.", + "bliss": "present:They dance at party night. " + }, + { + "original": "I graduate from college next year.", + "bliss": "present:I graduate from college year next. " + }, + { + "original": "We go on a road trip every summer.", + "bliss": "present:We go on road trip summer. " + }, + { + "original": "She paints a beautiful picture every night.", + "bliss": "present:She paint picture beautiful night. " + }, + { + "original": "He fixes the broken window every weekend.", + "bliss": "present:He fix window broken weekend. " + }, + { + "original": "They sing songs around the campfire every night.", + "bliss": "present:They sing around songs campfire night. " + }, + { + "original": "I wake up early every morning.", + "bliss": "present:I wake up morning. " + }, + { + "original": "We play board games all night every night.", + "bliss": "present:We play board games night night. " + }, + { + "original": "She bakes cookies for her friends every day.", + "bliss": "present:She bake cookies for her friends day. " + }, + { + "original": "He travels to Europe every summer.", + "bliss": "present:He travel to Europe summer. " + }, + { + "original": "They attend a concert every night.", + "bliss": "present:They attend concert night. " + }, + { + "original": "I finish reading the book every week.", + "bliss": "present:I finish read book week. " + }, + { + "original": "We go fishing at the lake every weekend.", + "bliss": "present:We go fish at lake weekend. " + }, + { + "original": "She practices the piano for hours every day.", + "bliss": "present:She practice piano for hours day. " + }, + { + "original": "He cleans his room every day.", + "bliss": "present:He clean his room day. " + }, + { + "original": "They plant flowers in the garden every spring.", + "bliss": "present:They plant flowers in garden spring. " + }, + { + "original": "I watch a documentary on TV every night.", + "bliss": "present:I watch on documentary TV night. " + }, + { + "original": "We have a barbecue in the backyard every weekend.", + "bliss": "present:We have barbecue in backyard weekend. " + }, + { + "original": "She studies for her exams every week.", + "bliss": "present:She study for her exams week. " + }, + { + "original": "He fixes the leaky faucet every weekend.", + "bliss": "present:He fix faucet leaky weekend. " + }, + { + "original": "They walk in the park every day.", + "bliss": "present:They walk in park day. " + }, + { + "original": "I play basketball with my friends every night.", + "bliss": "present:I play basketball with my friends night. " + }, + { + "original": "We visit the zoo every weekend.", + "bliss": "present:We visit zoo weekend. " + }, + { + "original": "She writes a letter to her pen pal every day.", + "bliss": "present:She write to letter her pen pal day. " + }, + { + "original": "He mows the lawn every week.", + "bliss": "present:He mow lawn week. " + }, + { + "original": "They go shopping at the mall every weekend.", + "bliss": "present:They go shop at mall weekend. " + }, + { + "original": "I go for a run every morning.", + "bliss": "present:I go for run morning. " + }, + { + "original": "We watch the sunset at the beach every night.", + "bliss": "present:We watch sunset at beach night. " + }, + { + "original": "She finishes knitting a sweater every week.", + "bliss": "present:She finish knit sweater week. " + }, + { + "original": "He cooks dinner for his family every day.", + "bliss": "present:He cook for dinner his family day. " + }, + { + "original": "They hike in the mountains every weekend.", + "bliss": "present:They hike in mountains weekend. " + }, + { + "original": "I clean out the garage every weekend.", + "bliss": "present:I clean out garage weekend. " + }, + { + "original": "We have a picnic in the park every weekend.", + "bliss": "present:We have in picnic park weekend. " + }, + { + "original": "She paints her nails every day.", + "bliss": "present:She paint her nails day. " + }, + { + "original": "He reads a book in the garden every day.", + "bliss": "present:He read book in garden day. " + }, + { + "original": "They play soccer in the park every day.", + "bliss": "present:They play soccer in park day. " + }, + { + "original": "I attend a party every weekend.", + "bliss": "present:I attend party weekend. " + }, + { + "original": "We ride bikes around the neighborhood every weekend.", + "bliss": "present:We ride around bikes neighborhood weekend. " + }, + { + "original": "She watches a movie with her friends every night.", + "bliss": "present:She watch movie with her friends night. " + }, + { + "original": "He goes for a swim at the pool every day.", + "bliss": "present:He go for at swim pool day. " + }, + { + "original": "They go for a hike in the forest every day.", + "bliss": "present:They go for in hike forest day. " + }, + { + "original": "I visit my aunt every weekend.", + "bliss": "present:I visit my aunt weekend. " + }, + { + "original": "We play video games all day every day.", + "bliss": "present:We play video games day day. " + }, + { + "original": "She cooks a delicious meal every day.", + "bliss": "present:She cook meal delicious day. " + }, + { + "original": "He rides his skateboard to the park every day.", + "bliss": "present:He ride his skateboard to park day. " + }, + { + "original": "They have a barbecue in the backyard every day.", + "bliss": "present:They have barbecue in backyard day. " + }, + { + "original": "I go camping with my family every weekend.", + "bliss": "present:I go camp with my family weekend. " + }, + { + "original": "We go to the beach every weekend.", + "bliss": "present:We go to beach weekend. " + }, + { + "original": "She practices yoga in the morning every day.", + "bliss": "present:She practice yoga in morning day. " + }, + { + "original": "He listens to music in his room every day.", + "bliss": "present:He listen to music in his room day. " + }, + { + "original": "They have a sleepover at their friend's house every weekend.", + "bliss": "present:They have sleepover at their friend house weekend. " + }, + { + "original": "I go for a bike ride every day.", + "bliss": "present:I go for bike ride day. " + }, + { + "original": "We go for a walk in the park every day.", + "bliss": "present:We go for in walk park day. " + }, + { + "original": "She takes a nap in the afternoon every day.", + "bliss": "present:She take nap in afternoon day. " + }, + { + "original": "He plays guitar for his friends every day.", + "bliss": "present:He play guitar for his friends day. " + }, + { + "original": "They go for a drive in the countryside every day.", + "bliss": "present:They go for in drive countryside day. " + }, + { + "original": "I visit the library every week.", + "bliss": "present:I visit library week. " + }, + { + "original": "We go to a party every night.", + "bliss": "present:We go to party night. " + }, + { + "original": "She swims in the pool every day.", + "bliss": "present:She swim in pool day. " + }, + { + "original": "He cooks breakfast for his family every day.", + "bliss": "present:He cook for breakfast his family day. " + }, + { + "original": "They watch a movie at home every day.", + "bliss": "present:They watch movie at home day. " + }, + { + "original": "I play video games with my brother every day.", + "bliss": "present:I play video games with my brother day. " + }, + { + "original": "We go to the amusement park every weekend.", + "bliss": "present:We go to amusement park weekend. " + }, + { + "original": "She goes shopping with her friends every day.", + "bliss": "present:She go shop with her friends day. " + }, + { + "original": "He rides his bike to school every day.", + "bliss": "present:He ride his bike to school day. " + }, + { + "original": "They visit their grandparents every weekend.", + "bliss": "present:They visit their grandparents weekend. " + }, + { + "original": "I clean my room every day.", + "bliss": "present:I clean my room day. " + }, + { + "original": "We have a picnic in the backyard every weekend.", + "bliss": "present:We have in picnic backyard weekend. " + }, + { + "original": "She reads a book in the garden every day.", + "bliss": "present:She read book in garden day. " + }, + { + "original": "He plays soccer with his friends every day.", + "bliss": "present:He play soccer with his friends day. " + }, + { + "original": "They have a barbecue at the beach every day.", + "bliss": "present:They have barbecue at beach day. " + }, + { + "original": "I go for a swim in the lake every day.", + "bliss": "present:I go for in swim lake day. " + }, + { + "original": "We watch the sunset at the park every weekend.", + "bliss": "present:We watch sunset at park weekend. " + }, + { + "original": "She goes hiking with her friends every day.", + "bliss": "present:She go hike with her friends day. " + }, + { + "original": "He rides his skateboard at the skate park every day.", + "bliss": "present:He ride his skateboard at skate park day. " + }, + { + "original": "They have a picnic in the forest every day.", + "bliss": "present:They have in picnic forest day. " + }, + { + "original": "I visit the museum every weekend.", + "bliss": "present:I visit museum weekend. " + }, + { + "original": "We go to a concert every night.", + "bliss": "present:We go to concert night. " + }, + { + "original": "She plays tennis with her friends every day.", + "bliss": "present:She play tennis with her friends day. " + }, + { + "original": "He goes fishing with his dad every day.", + "bliss": "present:He go fish with his dad day. " + }, + { + "original": "They have a barbecue in the park every day.", + "bliss": "present:They have in barbecue park day. " + }, + { + "original": "I go for a jog in the morning every day.", + "bliss": "present:I go for in jog morning day. " + }, + { + "original": "We have a bonfire at the beach every weekend.", + "bliss": "present:We have at bonfire beach weekend. " + }, + { + "original": "She goes for a bike ride in the countryside every day.", + "bliss": "present:She go for bike in ride countryside day. " + }, + { + "original": "She walks to the store every day.", + "bliss": "present:She walk to store day. " + }, + { + "original": "They play soccer after school.", + "bliss": "present:They play soccer after school. " + }, + { + "original": "He studies for his exam every night.", + "bliss": "present:He study for his exam night. " + }, + { + "original": "We visit Paris every summer.", + "bliss": "present:We visit Paris summer. " + }, + { + "original": "The cat chases the mouse around the house.", + "bliss": "present:cat chase mouse around house. " + }, + { + "original": "I finish my homework before dinner.", + "bliss": "present:I finish my homework before dinner. " + }, + { + "original": "She cooks dinner for her family.", + "bliss": "present:She cook for dinner her family. " + }, + { + "original": "They watch a movie at the cinema.", + "bliss": "present:They watch movie at cinema. " + }, + { + "original": "He runs a marathon every month.", + "bliss": "present:He run marathon month. " + }, + { + "original": "We go camping in the mountains.", + "bliss": "present:We go in camping mountains. " + }, + { + "original": "The sun sets over the horizon.", + "bliss": "present:sun set over horizon. " + }, + { + "original": "I read a book in bed every night.", + "bliss": "present:I read book in bed night. " + }, + { + "original": "She paints her bedroom walls blue.", + "bliss": "present:She paint her bedroom walls. " + }, + { + "original": "They visit their grandparents over the holidays.", + "bliss": "present:They visit their grandparents over holidays. " + }, + { + "original": "He fixes the leaky faucet in the bathroom.", + "bliss": "present:He fix in faucet leaky bathroom. " + }, + { + "original": "We ride bikes through the park.", + "bliss": "present:We ride bikes through park. " + }, + { + "original": "The children play in the backyard.", + "bliss": "present:children play in backyard. " + }, + { + "original": "She dances at her sister's wedding.", + "bliss": "present:She dance at her sister wedding. " + }, + { + "original": "They build a sandcastle at the beach.", + "bliss": "present:They build at sandcastle beach. " + }, + { + "original": "He wakes up early every morning.", + "bliss": "present:He wake up morning. " + }, + { + "original": "I eat breakfast before leaving for work.", + "bliss": "present:I eat breakfast before leave for work. " + }, + { + "original": "She swims in the pool for hours.", + "bliss": "present:She swim in pool for hours. " + }, + { + "original": "They visit the museum downtown.", + "bliss": "present:They visit museum downtown. " + }, + { + "original": "We hike up the mountain trail.", + "bliss": "present:We hike up mountain trail. " + }, + { + "original": "The dog barks at the mailman.", + "bliss": "present:dog bark at mailman. " + }, + { + "original": "I drive to the grocery store.", + "bliss": "present:I drive to grocery store. " + }, + { + "original": "She sings a song at the talent show.", + "bliss": "present:She sing at song talent show. " + }, + { + "original": "They fly to Europe for vacation.", + "bliss": "present:They fly to Europe for vacation. " + }, + { + "original": "He cooks dinner on the grill.", + "bliss": "present:He cook on dinner grill. " + }, + { + "original": "We watch fireworks on the Fourth of July.", + "bliss": "present:We watch of fireworks of Fourth July. " + }, + { + "original": "The baby sleeps through the night.", + "bliss": "present:baby sleep through night. " + }, + { + "original": "I complete the project ahead of schedule.", + "bliss": "present:I complete ahead project of schedule. " + }, + { + "original": "She knits a sweater for her friend.", + "bliss": "present:She knit sweater for her friend. " + }, + { + "original": "They take a road trip across the country.", + "bliss": "present:They take road across trip country. " + }, + { + "original": "He climbs to the top of the mountain.", + "bliss": "present:He climb to of top mountain. " + }, + { + "original": "We explore the ancient ruins.", + "bliss": "present:We explore ruins ancient. " + }, + { + "original": "The train arrives at the station.", + "bliss": "present:train arrive at station. " + }, + { + "original": "I finish reading the novel.", + "bliss": "present:I finish read novel. " + }, + { + "original": "She writes a letter to her pen pal.", + "bliss": "present:She write to letter her pen pal. " + }, + { + "original": "They visit the zoo on Saturday.", + "bliss": "present:They visit zoo on Saturday. " + }, + { + "original": "He fixes the broken window.", + "bliss": "present:He fix window broken. " + }, + { + "original": "We play board games by the fireplace.", + "bliss": "present:We play board games by fireplace. " + }, + { + "original": "The birds chirp in the trees.", + "bliss": "present:birds chirp in trees. " + }, + { + "original": "I ride my bike to school.", + "bliss": "present:I ride my bike to school. " + }, + { + "original": "She sews a dress for the school play.", + "bliss": "present:She sew for dress school play. " + }, + { + "original": "They plant flowers in the garden.", + "bliss": "present:They plant flowers in garden. " + }, + { + "original": "He visits his old high school.", + "bliss": "present:He visit his school old high. " + }, + { + "original": "We cook dinner together as a family.", + "bliss": "present:We cook together dinner as family. " + }, + { + "original": "The rain pours down all afternoon.", + "bliss": "present:rain pour down afternoon. " + }, + { + "original": "I bake cookies for the bake sale.", + "bliss": "present:I bake cookies for bake sale. " + }, + { + "original": "She walks to the store every day.", + "bliss": "present:She walk to store day. " + }, + { + "original": "They play soccer after school.", + "bliss": "present:They play soccer after school. " + }, + { + "original": "He studies for his exam every night.", + "bliss": "present:He study for his exam night. " + }, + { + "original": "We visit our grandparents every weekend.", + "bliss": "present:We visit our grandparents weekend. " + }, + { + "original": "The cat chases the mouse around the house.", + "bliss": "present:cat chase mouse around house. " + }, + { + "original": "I finish my homework before dinner.", + "bliss": "present:I finish my homework before dinner. " + }, + { + "original": "She cooks dinner for her family.", + "bliss": "present:She cook for dinner her family. " + }, + { + "original": "They watch a movie at the cinema.", + "bliss": "present:They watch movie at cinema. " + }, + { + "original": "He runs a marathon every month.", + "bliss": "present:He run marathon month. " + }, + { + "original": "We go swimming at the beach every summer.", + "bliss": "present:We go swim at beach summer. " + }, + { + "original": "The sun sets over the horizon.", + "bliss": "present:sun set over horizon. " + }, + { + "original": "I read a book in bed every night.", + "bliss": "present:I read book in bed night. " + }, + { + "original": "She paints her bedroom walls blue.", + "bliss": "present:She paint her bedroom walls. " + }, + { + "original": "They travel to Europe for vacation.", + "bliss": "present:They travel to Europe for vacation. " + }, + { + "original": "He fixes the leaky faucet in the bathroom.", + "bliss": "present:He fix in faucet leaky bathroom. " + }, + { + "original": "We hike up the mountain trail.", + "bliss": "present:We hike up mountain trail. " + }, + { + "original": "The children play in the backyard.", + "bliss": "present:children play in backyard. " + }, + { + "original": "She dances at her sister's wedding.", + "bliss": "present:She dance at her sister wedding. " + }, + { + "original": "They build a sandcastle at the beach.", + "bliss": "present:They build at sandcastle beach. " + }, + { + "original": "He wakes up early every morning.", + "bliss": "present:He wake up morning. " + }, + { + "original": "I take a bus to work every day.", + "bliss": "present:I take bus to work day. " + }, + { + "original": "She swims in the pool for hours.", + "bliss": "present:She swim in pool for hours. " + }, + { + "original": "They visit the museum downtown.", + "bliss": "present:They visit museum downtown. " + }, + { + "original": "We explore the ancient ruins.", + "bliss": "present:We explore ruins ancient. " + }, + { + "original": "The dog barks at the mailman.", + "bliss": "present:dog bark at mailman. " + }, + { + "original": "I drive to the grocery store.", + "bliss": "present:I drive to grocery store. " + }, + { + "original": "She sings a song at the talent show.", + "bliss": "present:She sing at song talent show. " + }, + { + "original": "They fly to Europe for vacation.", + "bliss": "present:They fly to Europe for vacation. " + }, + { + "original": "He cooks dinner on the grill.", + "bliss": "present:He cook on dinner grill. " + }, + { + "original": "We watch fireworks on the Fourth of July.", + "bliss": "present:We watch of fireworks of Fourth July. " + }, + { + "original": "The baby sleeps through the night.", + "bliss": "present:baby sleep through night. " + }, + { + "original": "I complete the project ahead of schedule.", + "bliss": "present:I complete ahead project of schedule. " + }, + { + "original": "She knits a sweater for her friend.", + "bliss": "present:She knit sweater for her friend. " + }, + { + "original": "They take a road trip across the country.", + "bliss": "present:They take road across trip country. " + }, + { + "original": "He climbs to the top of the mountain.", + "bliss": "present:He climb to of top mountain. " + }, + { + "original": "We bake cookies for the bake sale.", + "bliss": "present:We bake cookies for bake sale. " + }, + { + "original": "The rain pours down all afternoon.", + "bliss": "present:rain pour down afternoon. " + }, + { + "original": "I write a letter to my friend.", + "bliss": "present:I write letter to my friend. " + }, + { + "original": "She sews a dress for the school play.", + "bliss": "present:She sew for dress school play. " + }, + { + "original": "They plant flowers in the garden.", + "bliss": "present:They plant flowers in garden. " + }, + { + "original": "He visits his old high school.", + "bliss": "present:He visit his school old high. " + }, + { + "original": "We cook dinner together as a family.", + "bliss": "present:We cook together dinner as family. " + }, + { + "original": "The birds chirp in the trees.", + "bliss": "present:birds chirp in trees. " + }, + { + "original": "I ride my bike to school.", + "bliss": "present:I ride my bike to school. " + }, + { + "original": "She buys a new car.", + "bliss": "present:She buy car new. " + }, + { + "original": "They play board games by the fireplace.", + "bliss": "present:They play board games by fireplace. " + }, + { + "original": "He receives a gift from his parents.", + "bliss": "present:He receive gift from his parents. " + }, + { + "original": "We attend a concert.", + "bliss": "present:We attend concert. " + }, + { + "original": "The wind blows fiercely.", + "bliss": "present:wind blow fiercely. " + }, + { + "original": "I clean my room every day.", + "bliss": "present:I clean my room day. " + }, + { + "original": "She confidently walks to the nearby store every bright morning.", + "bliss": "present:She walk confidently to store nearby morning bright. " + }, + { + "original": "They enthusiastically play soccer after energetic school days.", + "bliss": "present:They play enthusiastically soccer after school days energetic. " + }, + { + "original": "He studiously studies for his important exam every quiet night.", + "bliss": "present:He study studiously for his exam important night quiet. " + }, + { + "original": "We regularly visit our loving grandparents every cozy weekend.", + "bliss": "present:We visit regularly our love grandparents weekend cozy. " + }, + { + "original": "The playful cat mischievously chases the startled mouse around the cozy house.", + "bliss": "present:cat playful chase mischievously mouse startled around house cozy. " + }, + { + "original": "I efficiently finish my challenging homework before delicious dinner.", + "bliss": "present:I finish efficiently my homework challenging before dinner delicious. " + }, + { + "original": "She skillfully cooks a delectable dinner for her grateful family.", + "bliss": "present:She cook skillfully for dinner delectable grateful her family grateful. " + }, + { + "original": "They quietly watch an entertaining movie at the nearby cinema.", + "bliss": "present:They watch quietly movie entertaining at cinema nearby. " + }, + { + "original": "He passionately runs a challenging marathon every active month.", + "bliss": "present:He run passionately marathon challenging month active. " + }, + { + "original": "We happily go swimming at the beautiful beach every sunny summer.", + "bliss": "present:We go happily swim at beach beautiful summer sunny. " + }, + { + "original": "The radiant sun peacefully sets over the breathtaking horizon.", + "bliss": "present:sun radiant set peacefully over horizon breathtaking. " + }, + { + "original": "I leisurely read an engaging book in bed every cozy night.", + "bliss": "past:I read leisurely engage book in bed night cozy. " + }, + { + "original": "She carefully paints her bedroom walls a soothing bright blue.", + "bliss": "present:She paint carefully her bedroom walls soothe blue bright. " + }, + { + "original": "They frequently travel to exciting Europe for relaxing vacation.", + "bliss": "present:They travel frequently to Europe for relax vacation. " + }, + { + "original": "He promptly fixes the leaky faucet in the gleaming bathroom.", + "bliss": "present:He fix promptly faucet leaky in gleam bathroom. " + }, + { + "original": "We eagerly hike up the scenic mountain trail every refreshing morning.", + "bliss": "present:We hike eagerly up mountain trail scenic morning refreshing. " + }, + { + "original": "The cheerful children happily play in the spacious backyard every sunny afternoon.", + "bliss": "present:children cheerful play happily in backyard spacious afternoon sunny. " + }, + { + "original": "She gracefully dances at her sister's enchanting wedding every festive occasion.", + "bliss": "present:She dance gracefully at her sister wedding enchanting occasion festive. " + }, + { + "original": "They creatively build a majestic sandcastle at the sandy beach every sunny day.", + "bliss": "present:They build creatively sandcastle majestic at beach sandy day sunny. " + }, + { + "original": "He wakes up early every misty morning to embrace the new day.", + "bliss": "present:He wake up morning misty to embrace day new. " + }, + { + "original": "I patiently take a crowded bus to work every bustling morning.", + "bliss": "present:I take patiently bus crowded to work morning bustling. " + }, + { + "original": "She blissfully swims in the refreshing pool for hours every hot afternoon.", + "bliss": "present:She swim blissfully in for pool refreshing hot hours afternoon hot. " + }, + { + "original": "They curiously visit the fascinating museum downtown every educational opportunity.", + "bliss": "present:They visit curiously museum downtown fascinating opportunity educational. " + }, + { + "original": "We bravely explore the ancient, mysterious ruins every adventurous weekend.", + "bliss": "present:We explore bravely ruins ancient mysterious adventurous weekend adventurous. " + }, + { + "original": "The vigilant dog fiercely barks at the unsuspecting mailman every busy morning.", + "bliss": "present:dog vigilant bark fiercely at mailman unsuspecting morning busy. " + }, + { + "original": "I cautiously drive to the bustling grocery store every busy weekend.", + "bliss": "present:I drive cautiously to grocery store bustling weekend busy. " + }, + { + "original": "She melodiously sings a beautiful song at the talent show every exciting event.", + "bliss": "present:She sing melodiously at song beautiful talent show event exciting. " + }, + { + "original": "They eagerly fly to romantic Europe for vacation every relaxing summer.", + "bliss": "present:They fly eagerly to Europe romantic for vacation relax summer. " + }, + { + "original": "He skillfully cooks a savory dinner on the sizzling grill every delicious evening.", + "bliss": "present:He cook skillfully on dinner savory sizzling grill sizzling evening delicious. " + }, + { + "original": "We joyfully watch spectacular fireworks on the Fourth of July every celebratory year.", + "bliss": "present:We watch joyfully of fireworks spectacular of Fourth July year celebratory. " + }, + { + "original": "The peaceful baby quietly sleeps through the calm night every cozy evening.", + "bliss": "present:baby peaceful sleep quietly through night calm evening cozy. " + }, + { + "original": "I efficiently complete the challenging project ahead of schedule every productive day.", + "bliss": "present:I complete efficiently ahead project challenging of schedule day productive. " + }, + { + "original": "She lovingly knits a warm sweater for her grateful friend every chilly season.", + "bliss": "present:She knit lovingly sweater warm for her friend grateful season chilly. " + }, + { + "original": "They leisurely take a scenic road trip across the vast country every adventurous summer.", + "bliss": "present:They take leisurely road across trip scenic vast country vast summer adventurous. " + }, + { + "original": "He courageously climbs to the breathtaking top of the tall mountain every adventurous weekend.", + "bliss": "present:He climb courageously to of top breathtaking tall mountain tall weekend adventurous. " + }, + { + "original": "We enthusiastically bake delicious cookies for the successful bake sale every festive occasion.", + "bliss": "present:We bake enthusiastically cookies delicious for bake sale successful occasion festive. " + }, + { + "original": "The relentless rain pours down heavily all afternoon every gloomy day.", + "bliss": "present:rain relentless pour heavily down afternoon day gloomy. " + }, + { + "original": "I quickly write a heartfelt letter to my distant friend every peaceful afternoon.", + "bliss": "present:I write quickly letter heartfelt to my friend distant afternoon peaceful. " + }, + { + "original": "She carefully sews a beautiful dress for the exciting school play every exciting occasion.", + "bliss": "present:She sew carefully for dress beautiful exciting exciting school play exciting exciting occasion exciting. " + }, + { + "original": "They lovingly plant colorful flowers in the flourishing garden every bright spring.", + "bliss": "present:They plant lovingly flowers colorful in flourish garden spring bright. " + }, + { + "original": "He nostalgically visits his old high school for the reunion every memorable year.", + "bliss": "present:He visit nostalgically his school old high for reunion year memorable. " + }, + { + "original": "We joyfully cook a hearty dinner together as a happy family every peaceful evening.", + "bliss": "present:We cook joyfully together dinner hearty as family happy evening peaceful. " + }, + { + "original": "The cheerful birds happily chirp in the lush, green trees every bright morning.", + "bliss": "present:birds cheerful chirp happily in trees lush green morning bright. " + }, + { + "original": "I cheerfully ride my trusty bike to school every sunny morning.", + "bliss": "present:I ride cheerfully my bike trusty to school morning sunny. " + }, + { + "original": "She excitedly buys a shiny new car every exciting year.", + "bliss": "present:She buy excitedly car shiny new year exciting. " + }, + { + "original": "They merrily play engaging board games by the cozy fireplace every cozy evening.", + "bliss": "present:They play merrily engage board by games cozy fireplace cozy evening cozy. " + }, + { + "original": "He gratefully receives a thoughtful gift from his loving parents every memorable occasion.", + "bliss": "present:He receive gratefully gift thoughtful from his love parents memorable occasion memorable. " + }, + { + "original": "We enthusiastically attend an entertaining concert every exciting weekend.", + "bliss": "present:We attend enthusiastically concert entertaining weekend exciting. " + }, + { + "original": "The strong wind fiercely blows through the dark, stormy night every restless evening.", + "bliss": "present:wind strong blow fiercely through night dark stormy evening restless. " + }, + { + "original": "I diligently clean my cluttered room every peaceful afternoon.", + "bliss": "present:I clean diligently my room cluttered afternoon peaceful." + } +] diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index cec53a7..08187d2 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -8,10 +8,11 @@ pipeline ) from peft import LoraConfig +# from peft import PeftModel from trl import SFTTrainer # The local directory with the model and the tokenizer -model_dir = "/home/cindyli/llama2/Llama-2-7b-hf" +model_dir = "/home/cindyli/projects/ctb-whkchun/s2_bliss_LLMs/Llama-2-7b-hf" # The instruction dataset to use dataset_name = "/home/cindyli/llama2/finetune/bliss.json" @@ -100,7 +101,7 @@ group_by_length = True # Save checkpoint every X updates steps -save_steps = 0 +save_steps = 25 # Log every X updates steps logging_steps = 25 @@ -119,31 +120,15 @@ device_map = {"": 0} -def create_prompt_formats(sample): - """ - Creates a formatted prompt template for an entry in the dataset - - :param sample: one dictionary from the dataset - """ - +# Create a formatted prompt template for an entry in the dataset +def format_prompt(sample): # Initialize static strings for the prompt template - INTRO_BLURB = "Below is an example that converts an English sentence to a structure in the Bliss language." - INPUT_KEY = "Original English sentence:" - RESPONSE_KEY = "Sentence in the Bliss language structure:" - - # Combine a prompt with the static strings - blurb = f"{INTRO_BLURB}" - input_context = f"{INPUT_KEY}\n{sample['original']}" if sample["original"] else None - response = f"{RESPONSE_KEY}\n{sample['bliss']}" - - # Create a list of prompt template elements - parts = [part for part in [blurb, input_context, response] if part] - - # Join prompt template elements into a single string to create the prompt template - formatted_prompt = "\n\n".join(parts) + instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" + input_key = "### Input:\n" + response_key = "### Response:\n" - # Store the formatted prompt template in a new key "text" - sample["text"] = formatted_prompt + # Format the sample + sample["text"] = f"{instruction}{input_key}{sample['original']}\n\n{response_key}{sample['bliss']}\n" return sample @@ -188,7 +173,7 @@ def create_prompt_formats(sample): print(f"Column names are: {dataset.column_names}") # Convert the data into prompts using the instructional template -dataset = dataset.map(create_prompt_formats) +dataset = dataset.map(format_prompt) print(dataset) print(dataset[0]) @@ -205,7 +190,7 @@ def create_prompt_formats(sample): # Set training parameters training_arguments = TrainingArguments( - output_dir=output_dir, + output_dir=f"{output_dir}-{num_train_epochs}epochs", num_train_epochs=num_train_epochs, per_device_train_batch_size=per_device_train_batch_size, gradient_accumulation_steps=gradient_accumulation_steps, @@ -220,7 +205,8 @@ def create_prompt_formats(sample): max_steps=max_steps, warmup_ratio=warmup_ratio, group_by_length=group_by_length, - lr_scheduler_type=lr_scheduler_type + lr_scheduler_type=lr_scheduler_type, + report_to="tensorboard" ) # Set supervised fine-tuning parameters @@ -239,12 +225,87 @@ def create_prompt_formats(sample): trainer.train() # Save trained model -trainer.model.save_pretrained(new_model) +trainer.model.save_pretrained(f"{new_model}-{num_train_epochs}epochs") print("Done with the fine-tuning.") -# Run text generation pipeline with our next model -prompt = "Converts this sentence to a structure in the Bliss language: I am a software developer." -pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=200) -result = pipe(f"{prompt}") -print(result[0]['generated_text']) +# Evaluate the new model + + +# Inference +def generate_text(instruction, input, model, tokenizer): + input_key = "### Input:\n" + response_key = "### Response:\n" + prompt = f"{instruction}{input_key}{input}\n\n{response_key}\n" + input_ids = tokenizer(prompt, return_tensors="pt", truncation=True).input_ids.cuda() + outputs = model.generate(input_ids=input_ids, max_new_tokens=100, do_sample=True, top_p=0.9, temperature=0.9) + print(f"Instruction: {instruction}\n") + print(f"Prompt: {input}\n") + print(f"Generated instruction: {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") + # pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=400) + # pipe_return = pipe(f"[INST] {prompt} [/INST]") + # print(f"## Prompt: {prompt}\n## Response:\n{pipe_return[0]['generated_text']}\n\n") + + +# Word predictions +def predict_words(prompt, model, tokenizer): + pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer) + predictions = pipe(prompt, max_length=20, num_return_sequences=3) + + # write results into the result file + print(f"## Prompt: {prompt}\n## Predictions:\n") + for prediction in predictions: + print(f"- {prediction['generated_text']}\n") + print("\n\n") + + +print("1. Inference\n") +instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" +input = "I am a programmer." +generate_text(instruction, input, model, tokenizer) + +input = "Joe will explore the picturesque landscapes of a charming countryside village tomorrow." +generate_text(instruction, input, model, tokenizer) + +input = "I had the pleasure of watching a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling." +generate_text(instruction, input, model, tokenizer) + +instruction = "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" +input = "past:The girl run in the park." +generate_text(instruction, input, model, tokenizer) + +input = "future:month next, I embark on an journey exciting to explore the cultures vibrant and landscapes breathtaking of Southeast Asia." +generate_text(instruction, input, model, tokenizer) + +print("2. Word Prediction\n\n") +prompt = "present: Joe be in hospital. He" +predict_words(prompt, model, tokenizer) + +prompt = "Tomorrow will be a beautiful day. Running" +predict_words(prompt, model, tokenizer) + +# Empty VRAM +del model +del trainer + +# # Merge the adapter weights into the base model then save. +# # See https://www.philschmid.de/instruction-tune-llama-2 +# # Reload model in FP16 and merge it with LoRA weights +# base_model = AutoModelForCausalLM.from_pretrained( +# model_dir, +# local_files_only=True, +# low_cpu_mem_usage=True, +# return_dict=True, +# torch_dtype=torch.float16, +# device_map=device_map, +# ) +# model = PeftModel.from_pretrained(base_model, new_model) +# model = model.merge_and_unload() + +# # Reload tokenizer to save it +# tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, trust_remote_code=True) +# tokenizer.pad_token = tokenizer.eos_token +# tokenizer.padding_side = "right" + +# model.save_pretrained(f"./{new_model}-{num_train_epochs}epochs") +# tokenizer.save_pretrained(f"./{new_model}-{num_train_epochs}epochs") diff --git a/jobs/Llama2/finetune/job_finetune_7b_hf.sh b/jobs/Llama2/finetune/job_finetune_7b_hf.sh index a5df7e4..b12de61 100644 --- a/jobs/Llama2/finetune/job_finetune_7b_hf.sh +++ b/jobs/Llama2/finetune/job_finetune_7b_hf.sh @@ -1,17 +1,16 @@ #!/bin/bash #SBATCH --job-name=llama2-finetune-7b-hf -#SBATCH --time 10-00:00 +#SBATCH --time 2-00:00 #SBATCH --nodes=1 #SBATCH --gpus-per-node=v100l:1 -#SBATCH --mem=128G +#SBATCH --mem=64G #SBATCH --ntasks-per-node=4 #SBATCH --cpus-per-task=4 -#SBATCH --account=ctb-whkchun +#SBATCH --account=def-whkchun #SBATCH --output=%x.o%j pip install --upgrade pip module load python/3.11.5 -python -V virtualenv --no-download $SLURM_TMPDIR/env source $SLURM_TMPDIR/.env/bin/activate @@ -19,12 +18,9 @@ source $SLURM_TMPDIR/.env/bin/activate pip install --upgrade pip module load StdEnv/2023 rust/1.70.0 arrow/14.0.1 gcc/12.3 -pip install --no-index transformers==4.36.2 accelerate==0.25.0 peft==0.5.0 bitsandbytes==0.40.2 +pip install --no-index transformers==4.36.2 accelerate==0.25.0 peft==0.5.0 bitsandbytes==0.42.0 tensorboard pip install datasets==2.17.0 trl pip install -r /home/cindyli/llama2/requirements-llama2.txt -python -V -pip list - -echo "Fine-tuning Llama2 from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." +echo "=== Fine-tuning Llama2 from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." python /home/cindyli/llama2/finetune/finetune_7b_hf.py diff --git a/jobs/Llama2/finetune/results/7b-1epochs.txt b/jobs/Llama2/finetune/results/7b-1epochs.txt new file mode 100644 index 0000000..a4019f2 --- /dev/null +++ b/jobs/Llama2/finetune/results/7b-1epochs.txt @@ -0,0 +1,328 @@ +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: pip in /home/cindyli/.local/lib/python3.11/site-packages (24.0+computecanada) +created virtual environment CPython3.11.5.final.0-64 in 13298ms + creator CPython3Posix(dest=/localscratch/cindyli.26307351.0/env, clear=False, no_vcs_ignore=False, global=False) + seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/home/cindyli/.local/share/virtualenv) + added seed packages: pip==23.3.2, setuptools==69.0.3, wheel==0.41.3 + activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator +/localscratch/spool/slurmd/job26307351/slurm_script: line 16: /localscratch/cindyli.26307351.0/.env/bin/activate: No such file or directory +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: pip in /home/cindyli/.local/lib/python3.11/site-packages (24.0+computecanada) + +Inactive Modules: + 1) libffi/3.3 2) sqlite/3.43.1 + +Due to MODULEPATH changes, the following have been reloaded: + 1) mii/1.1.2 2) python/3.11.5 + +The following have been reloaded with a version change: + 1) StdEnv/2020 => StdEnv/2023 5) libfabric/1.10.1 => libfabric/1.18.0 + 2) gcccore/.9.3.0 => gcccore/.12.3 6) openmpi/4.0.3 => openmpi/4.1.5 + 3) gentoo/2020 => gentoo/2023 7) ucx/1.8.0 => ucx/1.14.1 + 4) imkl/2020.1.217 => imkl/2023.2.0 + +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: transformers==4.36.2 in /home/cindyli/.local/lib/python3.11/site-packages (4.36.2+computecanada) +Requirement already satisfied: accelerate==0.25.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.25.0+computecanada) +Requirement already satisfied: peft==0.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.5.0+computecanada) +Requirement already satisfied: bitsandbytes==0.42.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.42.0+computecanada) +Requirement already satisfied: tensorboard in /home/cindyli/.local/lib/python3.11/site-packages (2.15.1+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from transformers==4.36.2) (3.12.2) +Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.20.2+computecanada) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (1.25.2+computecanada) +Requirement already satisfied: packaging>=20.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from transformers==4.36.2) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (6.0.1+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2023.8.8+computecanada) +Requirement already satisfied: requests in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2.31.0+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.4.1+computecanada) +Requirement already satisfied: tqdm>=4.27 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (4.66.1+computecanada) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate==0.25.0) (5.9.5+computecanada) +Requirement already satisfied: torch>=1.10.0 in /home/cindyli/.local/lib/python3.11/site-packages (from accelerate==0.25.0) (2.2.0+computecanada) +Requirement already satisfied: scipy in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from bitsandbytes==0.42.0) (1.11.2+computecanada) +Requirement already satisfied: absl-py>=0.4 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.1.0+computecanada) +Requirement already satisfied: grpcio>=1.48.2 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.57.0+computecanada) +Requirement already satisfied: google-auth<3,>=1.6.3 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.25.2+computecanada) +Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.2.0+computecanada) +Requirement already satisfied: markdown>=2.6.8 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.5.2+computecanada) +Requirement already satisfied: protobuf<4.24,>=3.19.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (4.23.4+computecanada) +Requirement already satisfied: setuptools>=41.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from tensorboard) (68.1.2) +Requirement already satisfied: six>1.9 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from tensorboard) (1.16.0+computecanada) +Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (0.7.0+computecanada) +Requirement already satisfied: werkzeug>=1.0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.0.1+computecanada) +Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.2+computecanada) +Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (0.3.0+computecanada) +Requirement already satisfied: rsa<5,>=3.1.4 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9+computecanada) +Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth-oauthlib<2,>=0.5->tensorboard) (1.3.1+computecanada) +Requirement already satisfied: fsspec>=2023.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (2023.10.0+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.1.3+computecanada) +Requirement already satisfied: MarkupSafe>=2.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from werkzeug>=1.0.1->tensorboard) (2.1.3+computecanada) +Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/cindyli/.local/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.1+computecanada) +Requirement already satisfied: oauthlib>=3.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard) (3.2.2+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.10.0->accelerate==0.25.0) (1.3.0+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: datasets==2.17.0 in /home/cindyli/.local/lib/python3.11/site-packages (2.17.0+computecanada) +Requirement already satisfied: trl in /home/cindyli/.local/lib/python3.11/site-packages (0.7.10) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from datasets==2.17.0) (3.12.2) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (1.25.2+computecanada) +Requirement already satisfied: pyarrow>=12.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/arrow/14.0.1/lib/python3.11/site-packages (from datasets==2.17.0) (14.0.1) +Requirement already satisfied: pyarrow-hotfix in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.6+computecanada) +Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.3.8+computecanada) +Requirement already satisfied: pandas in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: requests>=2.19.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (2.31.0+computecanada) +Requirement already satisfied: tqdm>=4.62.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (4.66.1+computecanada) +Requirement already satisfied: xxhash in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: multiprocess in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.70.16+computecanada) +Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets==2.17.0) (2023.10.0+computecanada) +Requirement already satisfied: aiohttp in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.9.1+computecanada) +Requirement already satisfied: huggingface-hub>=0.19.4 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.20.2+computecanada) +Requirement already satisfied: packaging in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (6.0.1+computecanada) +Requirement already satisfied: torch>=1.4.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (2.2.0+computecanada) +Requirement already satisfied: transformers>=4.31.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (4.36.2+computecanada) +Requirement already satisfied: accelerate in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.25.0+computecanada) +Requirement already satisfied: tyro>=0.5.11 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.7.2) +Requirement already satisfied: attrs>=17.3.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (23.1.0+computecanada) +Requirement already satisfied: multidict<7.0,>=4.5 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (6.0.5+computecanada) +Requirement already satisfied: yarl<2.0,>=1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.9.4) +Requirement already satisfied: frozenlist>=1.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.4.1+computecanada) +Requirement already satisfied: aiosignal>=1.1.2 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.3.1+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->datasets==2.17.0) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.1.3+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (2023.8.8+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.4.1+computecanada) +Requirement already satisfied: docstring-parser>=0.14.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (0.15) +Requirement already satisfied: rich>=11.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (13.7.0+computecanada) +Requirement already satisfied: shtab>=1.5.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (1.6.5) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate->trl) (5.9.5+computecanada) +Requirement already satisfied: python-dateutil>=2.8.2 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2.8.2+computecanada) +Requirement already satisfied: pytz>=2020.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: tzdata>=2022.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: six>=1.5 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets==2.17.0) (1.16.0+computecanada) +Requirement already satisfied: markdown-it-py>=2.2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (3.0.0+computecanada) +Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (2.16.1+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch>=1.4.0->trl) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.4.0->trl) (1.3.0+computecanada) +Requirement already satisfied: mdurl~=0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro>=0.5.11->trl) (0.1.2+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: torch in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.2.0+computecanada) +Requirement already satisfied: fairscale in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 2)) (0.4.12+computecanada) +Requirement already satisfied: fire in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (0.4.0+computecanada) +Requirement already satisfied: sentencepiece in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 4)) (0.1.99+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.12.2) +Requirement already satisfied: typing-extensions>=4.8.0 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (4.9.0+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.1.3+computecanada) +Requirement already satisfied: fsspec in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2023.10.0+computecanada) +Requirement already satisfied: numpy>=1.25 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.25.2+computecanada) +Requirement already satisfied: six in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (1.16.0+computecanada) +Requirement already satisfied: termcolor in /home/cindyli/.local/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) +=== Fine-tuning Llama2 from job 26307351 on nodes cdr2545. +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( + Loading checkpoint shards: 0%| | 0/2 [00:00 StdEnv/2023 5) libfabric/1.10.1 => libfabric/1.18.0 + 2) gcccore/.9.3.0 => gcccore/.12.3 6) openmpi/4.0.3 => openmpi/4.1.5 + 3) gentoo/2020 => gentoo/2023 7) ucx/1.8.0 => ucx/1.14.1 + 4) imkl/2020.1.217 => imkl/2023.2.0 + +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: transformers==4.36.2 in /home/cindyli/.local/lib/python3.11/site-packages (4.36.2+computecanada) +Requirement already satisfied: accelerate==0.25.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.25.0+computecanada) +Requirement already satisfied: peft==0.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.5.0+computecanada) +Requirement already satisfied: bitsandbytes==0.42.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.42.0+computecanada) +Requirement already satisfied: tensorboard in /home/cindyli/.local/lib/python3.11/site-packages (2.15.1+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from transformers==4.36.2) (3.12.2) +Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.20.2+computecanada) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (1.25.2+computecanada) +Requirement already satisfied: packaging>=20.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from transformers==4.36.2) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (6.0.1+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2023.8.8+computecanada) +Requirement already satisfied: requests in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2.31.0+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.4.1+computecanada) +Requirement already satisfied: tqdm>=4.27 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (4.66.1+computecanada) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate==0.25.0) (5.9.5+computecanada) +Requirement already satisfied: torch>=1.10.0 in /home/cindyli/.local/lib/python3.11/site-packages (from accelerate==0.25.0) (2.2.0+computecanada) +Requirement already satisfied: scipy in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from bitsandbytes==0.42.0) (1.11.2+computecanada) +Requirement already satisfied: absl-py>=0.4 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.1.0+computecanada) +Requirement already satisfied: grpcio>=1.48.2 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.57.0+computecanada) +Requirement already satisfied: google-auth<3,>=1.6.3 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.25.2+computecanada) +Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.2.0+computecanada) +Requirement already satisfied: markdown>=2.6.8 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.5.2+computecanada) +Requirement already satisfied: protobuf<4.24,>=3.19.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (4.23.4+computecanada) +Requirement already satisfied: setuptools>=41.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from tensorboard) (68.1.2) +Requirement already satisfied: six>1.9 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from tensorboard) (1.16.0+computecanada) +Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (0.7.0+computecanada) +Requirement already satisfied: werkzeug>=1.0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.0.1+computecanada) +Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.2+computecanada) +Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (0.3.0+computecanada) +Requirement already satisfied: rsa<5,>=3.1.4 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9+computecanada) +Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth-oauthlib<2,>=0.5->tensorboard) (1.3.1+computecanada) +Requirement already satisfied: fsspec>=2023.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (2023.10.0+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.1.3+computecanada) +Requirement already satisfied: MarkupSafe>=2.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from werkzeug>=1.0.1->tensorboard) (2.1.3+computecanada) +Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/cindyli/.local/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.1+computecanada) +Requirement already satisfied: oauthlib>=3.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard) (3.2.2+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.10.0->accelerate==0.25.0) (1.3.0+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: datasets==2.17.0 in /home/cindyli/.local/lib/python3.11/site-packages (2.17.0+computecanada) +Requirement already satisfied: trl in /home/cindyli/.local/lib/python3.11/site-packages (0.7.10) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from datasets==2.17.0) (3.12.2) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (1.25.2+computecanada) +Requirement already satisfied: pyarrow>=12.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/arrow/14.0.1/lib/python3.11/site-packages (from datasets==2.17.0) (14.0.1) +Requirement already satisfied: pyarrow-hotfix in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.6+computecanada) +Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.3.8+computecanada) +Requirement already satisfied: pandas in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: requests>=2.19.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (2.31.0+computecanada) +Requirement already satisfied: tqdm>=4.62.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (4.66.1+computecanada) +Requirement already satisfied: xxhash in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: multiprocess in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.70.16+computecanada) +Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets==2.17.0) (2023.10.0+computecanada) +Requirement already satisfied: aiohttp in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.9.1+computecanada) +Requirement already satisfied: huggingface-hub>=0.19.4 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.20.2+computecanada) +Requirement already satisfied: packaging in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (6.0.1+computecanada) +Requirement already satisfied: torch>=1.4.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (2.2.0+computecanada) +Requirement already satisfied: transformers>=4.31.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (4.36.2+computecanada) +Requirement already satisfied: accelerate in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.25.0+computecanada) +Requirement already satisfied: tyro>=0.5.11 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.7.2) +Requirement already satisfied: attrs>=17.3.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (23.1.0+computecanada) +Requirement already satisfied: multidict<7.0,>=4.5 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (6.0.5+computecanada) +Requirement already satisfied: yarl<2.0,>=1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.9.4) +Requirement already satisfied: frozenlist>=1.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.4.1+computecanada) +Requirement already satisfied: aiosignal>=1.1.2 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.3.1+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->datasets==2.17.0) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.1.3+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (2023.8.8+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.4.1+computecanada) +Requirement already satisfied: docstring-parser>=0.14.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (0.15) +Requirement already satisfied: rich>=11.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (13.7.0+computecanada) +Requirement already satisfied: shtab>=1.5.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (1.6.5) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate->trl) (5.9.5+computecanada) +Requirement already satisfied: python-dateutil>=2.8.2 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2.8.2+computecanada) +Requirement already satisfied: pytz>=2020.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: tzdata>=2022.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: six>=1.5 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets==2.17.0) (1.16.0+computecanada) +Requirement already satisfied: markdown-it-py>=2.2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (3.0.0+computecanada) +Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (2.16.1+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch>=1.4.0->trl) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.4.0->trl) (1.3.0+computecanada) +Requirement already satisfied: mdurl~=0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro>=0.5.11->trl) (0.1.2+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: torch in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.2.0+computecanada) +Requirement already satisfied: fairscale in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 2)) (0.4.12+computecanada) +Requirement already satisfied: fire in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (0.4.0+computecanada) +Requirement already satisfied: sentencepiece in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 4)) (0.1.99+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.12.2) +Requirement already satisfied: typing-extensions>=4.8.0 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (4.9.0+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.1.3+computecanada) +Requirement already satisfied: fsspec in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2023.10.0+computecanada) +Requirement already satisfied: numpy>=1.25 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.25.2+computecanada) +Requirement already satisfied: six in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (1.16.0+computecanada) +Requirement already satisfied: termcolor in /home/cindyli/.local/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) +=== Fine-tuning Llama2 from job 26309232 on nodes cdr2545. +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( + Loading checkpoint shards: 0%| | 0/2 [00:00 StdEnv/2023 5) libfabric/1.10.1 => libfabric/1.18.0 + 2) gcccore/.9.3.0 => gcccore/.12.3 6) openmpi/4.0.3 => openmpi/4.1.5 + 3) gentoo/2020 => gentoo/2023 7) ucx/1.8.0 => ucx/1.14.1 + 4) imkl/2020.1.217 => imkl/2023.2.0 + +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: transformers==4.36.2 in /home/cindyli/.local/lib/python3.11/site-packages (4.36.2+computecanada) +Requirement already satisfied: accelerate==0.25.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.25.0+computecanada) +Requirement already satisfied: peft==0.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.5.0+computecanada) +Requirement already satisfied: bitsandbytes==0.42.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.42.0+computecanada) +Requirement already satisfied: tensorboard in /home/cindyli/.local/lib/python3.11/site-packages (2.15.1+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from transformers==4.36.2) (3.12.2) +Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.20.2+computecanada) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (1.25.2+computecanada) +Requirement already satisfied: packaging>=20.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from transformers==4.36.2) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (6.0.1+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2023.8.8+computecanada) +Requirement already satisfied: requests in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2.31.0+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.4.1+computecanada) +Requirement already satisfied: tqdm>=4.27 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (4.66.1+computecanada) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate==0.25.0) (5.9.5+computecanada) +Requirement already satisfied: torch>=1.10.0 in /home/cindyli/.local/lib/python3.11/site-packages (from accelerate==0.25.0) (2.2.0+computecanada) +Requirement already satisfied: scipy in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from bitsandbytes==0.42.0) (1.11.2+computecanada) +Requirement already satisfied: absl-py>=0.4 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.1.0+computecanada) +Requirement already satisfied: grpcio>=1.48.2 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.57.0+computecanada) +Requirement already satisfied: google-auth<3,>=1.6.3 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.25.2+computecanada) +Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.2.0+computecanada) +Requirement already satisfied: markdown>=2.6.8 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.5.2+computecanada) +Requirement already satisfied: protobuf<4.24,>=3.19.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (4.23.4+computecanada) +Requirement already satisfied: setuptools>=41.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from tensorboard) (68.1.2) +Requirement already satisfied: six>1.9 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from tensorboard) (1.16.0+computecanada) +Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (0.7.0+computecanada) +Requirement already satisfied: werkzeug>=1.0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.0.1+computecanada) +Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.2+computecanada) +Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (0.3.0+computecanada) +Requirement already satisfied: rsa<5,>=3.1.4 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9+computecanada) +Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth-oauthlib<2,>=0.5->tensorboard) (1.3.1+computecanada) +Requirement already satisfied: fsspec>=2023.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (2023.10.0+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.1.3+computecanada) +Requirement already satisfied: MarkupSafe>=2.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from werkzeug>=1.0.1->tensorboard) (2.1.3+computecanada) +Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/cindyli/.local/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.1+computecanada) +Requirement already satisfied: oauthlib>=3.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard) (3.2.2+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.10.0->accelerate==0.25.0) (1.3.0+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: datasets==2.17.0 in /home/cindyli/.local/lib/python3.11/site-packages (2.17.0+computecanada) +Requirement already satisfied: trl in /home/cindyli/.local/lib/python3.11/site-packages (0.7.10) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from datasets==2.17.0) (3.12.2) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (1.25.2+computecanada) +Requirement already satisfied: pyarrow>=12.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/arrow/14.0.1/lib/python3.11/site-packages (from datasets==2.17.0) (14.0.1) +Requirement already satisfied: pyarrow-hotfix in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.6+computecanada) +Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.3.8+computecanada) +Requirement already satisfied: pandas in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: requests>=2.19.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (2.31.0+computecanada) +Requirement already satisfied: tqdm>=4.62.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (4.66.1+computecanada) +Requirement already satisfied: xxhash in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: multiprocess in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.70.16+computecanada) +Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets==2.17.0) (2023.10.0+computecanada) +Requirement already satisfied: aiohttp in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.9.1+computecanada) +Requirement already satisfied: huggingface-hub>=0.19.4 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.20.2+computecanada) +Requirement already satisfied: packaging in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (6.0.1+computecanada) +Requirement already satisfied: torch>=1.4.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (2.2.0+computecanada) +Requirement already satisfied: transformers>=4.31.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (4.36.2+computecanada) +Requirement already satisfied: accelerate in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.25.0+computecanada) +Requirement already satisfied: tyro>=0.5.11 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.7.2) +Requirement already satisfied: attrs>=17.3.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (23.1.0+computecanada) +Requirement already satisfied: multidict<7.0,>=4.5 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (6.0.5+computecanada) +Requirement already satisfied: yarl<2.0,>=1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.9.4) +Requirement already satisfied: frozenlist>=1.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.4.1+computecanada) +Requirement already satisfied: aiosignal>=1.1.2 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.3.1+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->datasets==2.17.0) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.1.3+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (2023.8.8+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.4.1+computecanada) +Requirement already satisfied: docstring-parser>=0.14.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (0.15) +Requirement already satisfied: rich>=11.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (13.7.0+computecanada) +Requirement already satisfied: shtab>=1.5.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (1.6.5) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate->trl) (5.9.5+computecanada) +Requirement already satisfied: python-dateutil>=2.8.2 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2.8.2+computecanada) +Requirement already satisfied: pytz>=2020.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: tzdata>=2022.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: six>=1.5 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets==2.17.0) (1.16.0+computecanada) +Requirement already satisfied: markdown-it-py>=2.2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (3.0.0+computecanada) +Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (2.16.1+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch>=1.4.0->trl) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.4.0->trl) (1.3.0+computecanada) +Requirement already satisfied: mdurl~=0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro>=0.5.11->trl) (0.1.2+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: torch in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.2.0+computecanada) +Requirement already satisfied: fairscale in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 2)) (0.4.12+computecanada) +Requirement already satisfied: fire in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (0.4.0+computecanada) +Requirement already satisfied: sentencepiece in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 4)) (0.1.99+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.12.2) +Requirement already satisfied: typing-extensions>=4.8.0 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (4.9.0+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.1.3+computecanada) +Requirement already satisfied: fsspec in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2023.10.0+computecanada) +Requirement already satisfied: numpy>=1.25 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.25.2+computecanada) +Requirement already satisfied: six in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (1.16.0+computecanada) +Requirement already satisfied: termcolor in /home/cindyli/.local/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) +=== Fine-tuning Llama2 from job 26309233 on nodes cdr2545. +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( + Loading checkpoint shards: 0%| | 0/2 [00:00 StdEnv/2023 5) libfabric/1.10.1 => libfabric/1.18.0 + 2) gcccore/.9.3.0 => gcccore/.12.3 6) openmpi/4.0.3 => openmpi/4.1.5 + 3) gentoo/2020 => gentoo/2023 7) ucx/1.8.0 => ucx/1.14.1 + 4) imkl/2020.1.217 => imkl/2023.2.0 + +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: transformers==4.36.2 in /home/cindyli/.local/lib/python3.11/site-packages (4.36.2+computecanada) +Requirement already satisfied: accelerate==0.25.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.25.0+computecanada) +Requirement already satisfied: peft==0.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.5.0+computecanada) +Requirement already satisfied: bitsandbytes==0.42.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.42.0+computecanada) +Requirement already satisfied: tensorboard in /home/cindyli/.local/lib/python3.11/site-packages (2.15.1+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from transformers==4.36.2) (3.12.2) +Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.20.2+computecanada) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (1.25.2+computecanada) +Requirement already satisfied: packaging>=20.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from transformers==4.36.2) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (6.0.1+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2023.8.8+computecanada) +Requirement already satisfied: requests in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2.31.0+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.4.1+computecanada) +Requirement already satisfied: tqdm>=4.27 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (4.66.1+computecanada) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate==0.25.0) (5.9.5+computecanada) +Requirement already satisfied: torch>=1.10.0 in /home/cindyli/.local/lib/python3.11/site-packages (from accelerate==0.25.0) (2.2.0+computecanada) +Requirement already satisfied: scipy in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from bitsandbytes==0.42.0) (1.11.2+computecanada) +Requirement already satisfied: absl-py>=0.4 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.1.0+computecanada) +Requirement already satisfied: grpcio>=1.48.2 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.57.0+computecanada) +Requirement already satisfied: google-auth<3,>=1.6.3 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (2.25.2+computecanada) +Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (1.2.0+computecanada) +Requirement already satisfied: markdown>=2.6.8 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.5.2+computecanada) +Requirement already satisfied: protobuf<4.24,>=3.19.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (4.23.4+computecanada) +Requirement already satisfied: setuptools>=41.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from tensorboard) (68.1.2) +Requirement already satisfied: six>1.9 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from tensorboard) (1.16.0+computecanada) +Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (0.7.0+computecanada) +Requirement already satisfied: werkzeug>=1.0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tensorboard) (3.0.1+computecanada) +Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.2+computecanada) +Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (0.3.0+computecanada) +Requirement already satisfied: rsa<5,>=3.1.4 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9+computecanada) +Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from google-auth-oauthlib<2,>=0.5->tensorboard) (1.3.1+computecanada) +Requirement already satisfied: fsspec>=2023.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (2023.10.0+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.2) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.10.0->accelerate==0.25.0) (3.1.3+computecanada) +Requirement already satisfied: MarkupSafe>=2.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from werkzeug>=1.0.1->tensorboard) (2.1.3+computecanada) +Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/cindyli/.local/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.1+computecanada) +Requirement already satisfied: oauthlib>=3.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard) (3.2.2+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.10.0->accelerate==0.25.0) (1.3.0+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: datasets==2.17.0 in /home/cindyli/.local/lib/python3.11/site-packages (2.17.0+computecanada) +Requirement already satisfied: trl in /home/cindyli/.local/lib/python3.11/site-packages (0.7.10) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from datasets==2.17.0) (3.12.2) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (1.25.2+computecanada) +Requirement already satisfied: pyarrow>=12.0.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/arrow/14.0.1/lib/python3.11/site-packages (from datasets==2.17.0) (14.0.1) +Requirement already satisfied: pyarrow-hotfix in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.6+computecanada) +Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.3.8+computecanada) +Requirement already satisfied: pandas in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: requests>=2.19.0 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (2.31.0+computecanada) +Requirement already satisfied: tqdm>=4.62.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (4.66.1+computecanada) +Requirement already satisfied: xxhash in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: multiprocess in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.70.16+computecanada) +Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets==2.17.0) (2023.10.0+computecanada) +Requirement already satisfied: aiohttp in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (3.9.1+computecanada) +Requirement already satisfied: huggingface-hub>=0.19.4 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (0.20.2+computecanada) +Requirement already satisfied: packaging in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from datasets==2.17.0) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from datasets==2.17.0) (6.0.1+computecanada) +Requirement already satisfied: torch>=1.4.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (2.2.0+computecanada) +Requirement already satisfied: transformers>=4.31.0 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (4.36.2+computecanada) +Requirement already satisfied: accelerate in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.25.0+computecanada) +Requirement already satisfied: tyro>=0.5.11 in /home/cindyli/.local/lib/python3.11/site-packages (from trl) (0.7.2) +Requirement already satisfied: attrs>=17.3.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (23.1.0+computecanada) +Requirement already satisfied: multidict<7.0,>=4.5 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (6.0.5+computecanada) +Requirement already satisfied: yarl<2.0,>=1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.9.4) +Requirement already satisfied: frozenlist>=1.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.4.1+computecanada) +Requirement already satisfied: aiosignal>=1.1.2 in /home/cindyli/.local/lib/python3.11/site-packages (from aiohttp->datasets==2.17.0) (1.3.1+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->datasets==2.17.0) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests>=2.19.0->datasets==2.17.0) (2023.7.22+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.4.0->trl) (3.1.3+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (2023.8.8+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers>=4.31.0->trl) (0.4.1+computecanada) +Requirement already satisfied: docstring-parser>=0.14.1 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (0.15) +Requirement already satisfied: rich>=11.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (13.7.0+computecanada) +Requirement already satisfied: shtab>=1.5.6 in /home/cindyli/.local/lib/python3.11/site-packages (from tyro>=0.5.11->trl) (1.6.5) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from accelerate->trl) (5.9.5+computecanada) +Requirement already satisfied: python-dateutil>=2.8.2 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2.8.2+computecanada) +Requirement already satisfied: pytz>=2020.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: tzdata>=2022.1 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/scipy-stack/2023b/lib/python3.11/site-packages (from pandas->datasets==2.17.0) (2023.3+computecanada) +Requirement already satisfied: six>=1.5 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets==2.17.0) (1.16.0+computecanada) +Requirement already satisfied: markdown-it-py>=2.2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (3.0.0+computecanada) +Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from rich>=11.1.0->tyro>=0.5.11->trl) (2.16.1+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch>=1.4.0->trl) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.4.0->trl) (1.3.0+computecanada) +Requirement already satisfied: mdurl~=0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro>=0.5.11->trl) (0.1.2+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: torch in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.2.0+computecanada) +Requirement already satisfied: fairscale in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 2)) (0.4.12+computecanada) +Requirement already satisfied: fire in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (0.4.0+computecanada) +Requirement already satisfied: sentencepiece in /home/cindyli/.local/lib/python3.11/site-packages (from -r /home/cindyli/llama2/requirements-llama2.txt (line 4)) (0.1.99+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.12.2) +Requirement already satisfied: typing-extensions>=4.8.0 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (4.9.0+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (3.1.3+computecanada) +Requirement already satisfied: fsspec in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2023.10.0+computecanada) +Requirement already satisfied: numpy>=1.25 in /home/cindyli/.local/lib/python3.11/site-packages (from torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.25.2+computecanada) +Requirement already satisfied: six in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (1.16.0+computecanada) +Requirement already satisfied: termcolor in /home/cindyli/.local/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) +=== Fine-tuning Llama2 from job 26309234 on nodes cdr2545. +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( + Loading checkpoint shards: 0%| | 0/2 [00:00[INST] {prompt} [/INST]") - -# write results into the result file -file_path = "/home/cindyli/llama2/original_use/result.txt" -file = open(file_path, "w") -file.write(f"1. Text generation: \n{pipe_return[0]['generated_text']}\n\n") - -# 2. Test prompt for word predictions -prompt = "David is feeling sick and in the hospital. He wants to" -pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer) -predictions = pipe(prompt, max_length=20, num_return_sequences=3) - -# write results into the result file -file.write("2. Word prediction: \n") -for prediction in predictions: - file.write(f"- {prediction['generated_text']}\n") -file.write("\n") - -# 3. Test prompt for inference -prompt = "David wants to" -inference = tokenizer.decode(model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_length=300)[0]) - -# write results into the result file -file.write(f"3. Inference: \n{inference}\n") + +# Text generation +def generate_text(prompt, output_file, model, tokenizer): + pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=400) + pipe_return = pipe(f"[INST] {prompt} [/INST]") + + # write results into the result file + output_file.write(f"## Text generation: \n{pipe_return[0]['generated_text']}\n\n") + + +# Word predictions +def predict_words(prompt, output_file, model, tokenizer): + pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer) + predictions = pipe(prompt, max_length=20, num_return_sequences=3) + + # write results into the result file + output_file.write("## Word prediction: \n") + for prediction in predictions: + output_file.write(f"- {prediction['generated_text']}\n") + output_file.write("\n") + + +# Generate inference +def get_inference(prompt, output_file, model, tokenizer): + inference = tokenizer.decode(model.generate(**tokenizer(prompt, return_tensors="pt").to(model.device), max_length=300)[0]) + + # write results into the result file + output_file.write(f"## Inference: \n{inference}\n") + + +instruction = "Convert this English sentence to a structure in the Bliss language: " +generate_text(f"{instruction}The government says the measures are meant to address the cost of living and spur home building in the province while critics say the spending is reckless.", output_file, model, tokenizer) +generate_text(f"{instruction}Today is a lovely sunny day.", output_file, model, tokenizer) +generate_text(f"{instruction}Yesterday, I watched a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling.", output_file, model, tokenizer) +generate_text(f"{instruction}I will explore the picturesque landscapes of a charming countryside village.", output_file, model, tokenizer) + +# predict_words("Joe is feeling sick and in the hospital. He wants to", output_file, model, tokenizer) +# get_inference("Joe wants to", output_file, model, tokenizer) From 9823afab04eb3cf2368be4f1bc8393a42f9f8967 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 5 Mar 2024 11:52:17 -0500 Subject: [PATCH 06/20] feat: add evaluation job/script and improve fine-tuning script --- jobs/Llama2/finetune/eval_7b_hf.py | 48 ++++++++++++++++++++++++++ jobs/Llama2/finetune/finetune_7b_hf.py | 25 -------------- jobs/Llama2/finetune/job_eval_7b_hf.sh | 24 +++++++++++++ 3 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 jobs/Llama2/finetune/eval_7b_hf.py create mode 100644 jobs/Llama2/finetune/job_eval_7b_hf.sh diff --git a/jobs/Llama2/finetune/eval_7b_hf.py b/jobs/Llama2/finetune/eval_7b_hf.py new file mode 100644 index 0000000..b26c816 --- /dev/null +++ b/jobs/Llama2/finetune/eval_7b_hf.py @@ -0,0 +1,48 @@ +import torch +from peft import AutoPeftModelForCausalLM +from transformers import AutoTokenizer + +model_dir = "/home/cindyli/llama2/finetune/results-finetune-7b-hf-4epochs/checkpoint-950/" + +# load base LLM model and tokenizer +model = AutoPeftModelForCausalLM.from_pretrained( + model_dir, + low_cpu_mem_usage=True, + torch_dtype=torch.float16, + load_in_4bit=True, + local_files_only=True +) +tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True) + + +# Evaluation +# Inference. Generated texts by giving temperature 0.1 to 0.9 in the step of 0.1 +def generate_text(instruction, input, model, tokenizer): + input_key = "### Input:\n" + response_key = "### Response:\n" + prompt = f"{instruction}{input_key}{input}\n\n{response_key}\n" + input_ids = tokenizer(prompt, return_tensors="pt", truncation=True).input_ids.cuda() + print(f"Instruction: {instruction}\n") + print(f"Prompt: {input}\n") + for temperature in range(0.1, 0.9, 0.1): + outputs = model.generate(input_ids=input_ids, max_new_tokens=100, do_sample=True, top_p=0.9, temperature=temperature) + print(f"Generated instruction (temprature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") + + +print("1. Inference\n") +instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" +input = "I am a programmer." +generate_text(instruction, input, model, tokenizer) + +input = "Joe will explore the picturesque landscapes of a charming countryside village tomorrow." +generate_text(instruction, input, model, tokenizer) + +input = "I had the pleasure of watching a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling." +generate_text(instruction, input, model, tokenizer) + +instruction = "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" +input = "past:The girl run in the park." +generate_text(instruction, input, model, tokenizer) + +input = "future:month next, I embark on an journey exciting to explore the cultures vibrant and landscapes breathtaking of Southeast Asia." +generate_text(instruction, input, model, tokenizer) diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 08187d2..12d0883 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -242,9 +242,6 @@ def generate_text(instruction, input, model, tokenizer): print(f"Instruction: {instruction}\n") print(f"Prompt: {input}\n") print(f"Generated instruction: {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") - # pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=400) - # pipe_return = pipe(f"[INST] {prompt} [/INST]") - # print(f"## Prompt: {prompt}\n## Response:\n{pipe_return[0]['generated_text']}\n\n") # Word predictions @@ -287,25 +284,3 @@ def predict_words(prompt, model, tokenizer): # Empty VRAM del model del trainer - -# # Merge the adapter weights into the base model then save. -# # See https://www.philschmid.de/instruction-tune-llama-2 -# # Reload model in FP16 and merge it with LoRA weights -# base_model = AutoModelForCausalLM.from_pretrained( -# model_dir, -# local_files_only=True, -# low_cpu_mem_usage=True, -# return_dict=True, -# torch_dtype=torch.float16, -# device_map=device_map, -# ) -# model = PeftModel.from_pretrained(base_model, new_model) -# model = model.merge_and_unload() - -# # Reload tokenizer to save it -# tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, trust_remote_code=True) -# tokenizer.pad_token = tokenizer.eos_token -# tokenizer.padding_side = "right" - -# model.save_pretrained(f"./{new_model}-{num_train_epochs}epochs") -# tokenizer.save_pretrained(f"./{new_model}-{num_train_epochs}epochs") diff --git a/jobs/Llama2/finetune/job_eval_7b_hf.sh b/jobs/Llama2/finetune/job_eval_7b_hf.sh new file mode 100644 index 0000000..201ae4d --- /dev/null +++ b/jobs/Llama2/finetune/job_eval_7b_hf.sh @@ -0,0 +1,24 @@ +#!/bin/bash +#SBATCH --job-name=llama2-finetune-7b-hf +#SBATCH --time 2-00:00 +#SBATCH --nodes=1 +#SBATCH --gpus-per-node=v100l:1 +#SBATCH --mem=64G +#SBATCH --ntasks-per-node=4 +#SBATCH --cpus-per-task=4 +#SBATCH --account=def-whkchun +#SBATCH --output=%x.o%j + +pip install --upgrade pip +module load python/3.11.5 + +virtualenv --no-download $SLURM_TMPDIR/env +source $SLURM_TMPDIR/.env/bin/activate + +pip install --upgrade pip + +module load StdEnv/2023 rust/1.70.0 arrow/14.0.1 gcc/12.3 +pip install --no-index torch transformers==4.36.2 peft==0.5.0 + +echo "=== Fine-tuning Llama2 from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." +python /home/cindyli/llama2/finetune/eval_7b_hf.py From 45bfe66a1ec03de21faf99375836f0081e84e138 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 6 Mar 2024 16:14:49 -0500 Subject: [PATCH 07/20] fix: fine-tune llama2 with bi-directional Bliss/English dataset --- jobs/Llama2/finetune/finetune_7b_hf.py | 35 ++- jobs/Llama2/finetune/results/7b-1epochs.txt | 165 ++++++------ jobs/Llama2/finetune/results/7b-2epochs.txt | 175 ++++++------ jobs/Llama2/finetune/results/7b-3epochs.txt | 283 ++++++++++++-------- jobs/Llama2/finetune/results/7b-4epochs.txt | 230 ++++++++-------- 5 files changed, 501 insertions(+), 387 deletions(-) diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 12d0883..399b2b3 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -1,5 +1,5 @@ import torch -from datasets import load_dataset +from datasets import load_dataset, concatenate_datasets from transformers import ( AutoModelForCausalLM, AutoTokenizer, @@ -18,7 +18,7 @@ dataset_name = "/home/cindyli/llama2/finetune/bliss.json" # Output directory where the model checkpoints will be stored -output_dir = "/home/cindyli/llama2/finetune/results-finetune-7b-hf" +output_dir = "/home/cindyli/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf" # Fine-tuned model name new_model = "llama-2-7b-hf-bliss" @@ -121,18 +121,22 @@ # Create a formatted prompt template for an entry in the dataset -def format_prompt(sample): +# The "direction" parameter accepts either "EnglishToBliss" or "BlissToEnglish" +def format_prompt(sample, direction="EnglishToBliss"): # Initialize static strings for the prompt template - instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" + instruction = f"### Instruction: \nConvert the input {'English' if direction=='EnglishToBliss' else 'Bliss'} sentence to a {'Bliss' if direction=='EnglishToBliss' else 'English'} sentence.\n\n" input_key = "### Input:\n" response_key = "### Response:\n" - # Format the sample - sample["text"] = f"{instruction}{input_key}{sample['original']}\n\n{response_key}{sample['bliss']}\n" + if direction == "EnglishToBliss": + # Format the sample for English to Bliss conversion + sample["text"] = f"{instruction}{input_key}{sample['original']}\n\n{response_key}{sample['bliss']}\n" + elif direction == "BlissToEnglish": + # Format the sample for Bliss to English conversion + sample["text"] = f"{instruction}{input_key}{sample['bliss']}\n\n{response_key}{sample['original']}\n" return sample - # Load tokenizer and model with QLoRA configuration compute_dtype = getattr(torch, bnb_4bit_compute_dtype) @@ -168,15 +172,20 @@ def format_prompt(sample): # Preprocess dataset print("Preprocessing dataset...") -dataset = load_dataset("json", data_files=dataset_name, split="train") -print(f"Number of prompts: {len(dataset)}") -print(f"Column names are: {dataset.column_names}") +orig_dataset = load_dataset("json", data_files=dataset_name, split="train") +print(f"Number of prompts: {len(orig_dataset)}") +print(f"Column names are: {orig_dataset.column_names}") + +# Create two datasets for English to Bliss and Bliss to English +english_to_bliss_dataset = orig_dataset.map(lambda x: format_prompt(x, direction="EnglishToBliss")) +bliss_to_english_dataset = orig_dataset.map(lambda x: format_prompt(x, direction="BlissToEnglish")) -# Convert the data into prompts using the instructional template -dataset = dataset.map(format_prompt) +# Combine the two datasets +dataset = concatenate_datasets([english_to_bliss_dataset, bliss_to_english_dataset]) print(dataset) -print(dataset[0]) +print(f"First record: {dataset[0]}") +print(f"Last record: {dataset[-1]}") print("Done with preprocessing dataset.\n\nStart fine-tuning...") # Load LoRA configuration diff --git a/jobs/Llama2/finetune/results/7b-1epochs.txt b/jobs/Llama2/finetune/results/7b-1epochs.txt index a4019f2..74a0530 100644 --- a/jobs/Llama2/finetune/results/7b-1epochs.txt +++ b/jobs/Llama2/finetune/results/7b-1epochs.txt @@ -1,12 +1,12 @@ Defaulting to user installation because normal site-packages is not writeable Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic Requirement already satisfied: pip in /home/cindyli/.local/lib/python3.11/site-packages (24.0+computecanada) -created virtual environment CPython3.11.5.final.0-64 in 13298ms - creator CPython3Posix(dest=/localscratch/cindyli.26307351.0/env, clear=False, no_vcs_ignore=False, global=False) +created virtual environment CPython3.11.5.final.0-64 in 9305ms + creator CPython3Posix(dest=/localscratch/cindyli.26441224.0/env, clear=False, no_vcs_ignore=False, global=False) seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/home/cindyli/.local/share/virtualenv) added seed packages: pip==23.3.2, setuptools==69.0.3, wheel==0.41.3 activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator -/localscratch/spool/slurmd/job26307351/slurm_script: line 16: /localscratch/cindyli.26307351.0/.env/bin/activate: No such file or directory +/localscratch/spool/slurmd/job26441224/slurm_script: line 16: /localscratch/cindyli.26441224.0/.env/bin/activate: No such file or directory Defaulting to user installation because normal site-packages is not writeable Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic Requirement already satisfied: pip in /home/cindyli/.local/lib/python3.11/site-packages (24.0+computecanada) @@ -139,50 +139,62 @@ Requirement already satisfied: six in /cvmfs/soft.computecanada.ca/easybuild/sof Requirement already satisfied: termcolor in /home/cindyli/.local/lib/python3.11/site-packages (from fire->-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) -=== Fine-tuning Llama2 from job 26307351 on nodes cdr2545. +=== Fine-tuning Llama2 from job 26441224 on nodes cdr2545. /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( - Loading checkpoint shards: 0%| | 0/2 [00:00-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) -=== Fine-tuning Llama2 from job 26309232 on nodes cdr2545. +=== Fine-tuning Llama2 from job 26469134 on nodes cdr2562. /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( - Loading checkpoint shards: 0%| | 0/2 [00:00-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) -=== Fine-tuning Llama2 from job 26309233 on nodes cdr2545. +=== Fine-tuning Llama2 from job 26477429 on nodes cdr2535. /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( - Loading checkpoint shards: 0%| | 0/2 [00:00-r /home/cindyli/llama2/requirements-llama2.txt (line 3)) (2.4.0+computecanada) Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (2.1.3+computecanada) Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch->-r /home/cindyli/llama2/requirements-llama2.txt (line 1)) (1.3.0+computecanada) -=== Fine-tuning Llama2 from job 26309234 on nodes cdr2545. +=== Fine-tuning Llama2 from job 26469135 on nodes cdr2549. /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( - Loading checkpoint shards: 0%| | 0/2 [00:00 Date: Wed, 6 Mar 2024 16:19:25 -0500 Subject: [PATCH 08/20] fix: linted --- jobs/Llama2/finetune/finetune_7b_hf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 399b2b3..c0fa05f 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -137,6 +137,7 @@ def format_prompt(sample, direction="EnglishToBliss"): return sample + # Load tokenizer and model with QLoRA configuration compute_dtype = getattr(torch, bnb_4bit_compute_dtype) From 2806452c2b920f346e6b9c30f949788a4b74e0a0 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 11 Mar 2024 13:54:49 -0400 Subject: [PATCH 09/20] fix: evaluate more random prompts --- jobs/Llama2/finetune/eval_7b_hf.py | 50 +++++-- .../results/7b-3epochs-other-prompts.txt | 130 ++++++++++++++++++ 2 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt diff --git a/jobs/Llama2/finetune/eval_7b_hf.py b/jobs/Llama2/finetune/eval_7b_hf.py index b26c816..64ecd88 100644 --- a/jobs/Llama2/finetune/eval_7b_hf.py +++ b/jobs/Llama2/finetune/eval_7b_hf.py @@ -2,7 +2,7 @@ from peft import AutoPeftModelForCausalLM from transformers import AutoTokenizer -model_dir = "/home/cindyli/llama2/finetune/results-finetune-7b-hf-4epochs/checkpoint-950/" +model_dir = "/home/cindyli/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf-3epochs/checkpoint-975" # load base LLM model and tokenizer model = AutoPeftModelForCausalLM.from_pretrained( @@ -16,33 +16,59 @@ # Evaluation -# Inference. Generated texts by giving temperature 0.1 to 0.9 in the step of 0.1 -def generate_text(instruction, input, model, tokenizer): +# Inference. Generated texts by giving an instruction +def generate_text_with_instruction(instruction, input, model, tokenizer, temperature=0.7): input_key = "### Input:\n" response_key = "### Response:\n" prompt = f"{instruction}{input_key}{input}\n\n{response_key}\n" input_ids = tokenizer(prompt, return_tensors="pt", truncation=True).input_ids.cuda() print(f"Instruction: {instruction}\n") print(f"Prompt: {input}\n") - for temperature in range(0.1, 0.9, 0.1): - outputs = model.generate(input_ids=input_ids, max_new_tokens=100, do_sample=True, top_p=0.9, temperature=temperature) - print(f"Generated instruction (temprature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") + outputs = model.generate(input_ids=input_ids, max_new_tokens=100, do_sample=True, top_p=0.9, temperature=temperature) + print(f"Generated instruction (temprature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") -print("1. Inference\n") +def generate_text_with_prompt(prompt, model, tokenizer, temperature=0.7): + input_ids = tokenizer(prompt, return_tensors="pt", truncation=True).input_ids.cuda() + print(f"Prompt: {prompt}\n") + outputs = model.generate(input_ids=input_ids, max_new_tokens=100, do_sample=True, top_p=0.9, temperature=temperature) + print(f"Generated instruction (temprature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") + + +# Test with exactly same instructions used in fine-tuning instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" input = "I am a programmer." -generate_text(instruction, input, model, tokenizer) +generate_text_with_instruction(instruction, input, model, tokenizer) input = "Joe will explore the picturesque landscapes of a charming countryside village tomorrow." -generate_text(instruction, input, model, tokenizer) +generate_text_with_instruction(instruction, input, model, tokenizer) input = "I had the pleasure of watching a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling." -generate_text(instruction, input, model, tokenizer) +generate_text_with_instruction(instruction, input, model, tokenizer) instruction = "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" input = "past:The girl run in the park." -generate_text(instruction, input, model, tokenizer) +generate_text_with_instruction(instruction, input, model, tokenizer) input = "future:month next, I embark on an journey exciting to explore the cultures vibrant and landscapes breathtaking of Southeast Asia." -generate_text(instruction, input, model, tokenizer) +generate_text_with_instruction(instruction, input, model, tokenizer) + +# Test with random propmts +# Two prompts below is copied from the dataset +prompt = "Convert this sentence to a Bliss sentence: He rode his skateboard at the skate park yesterday.\n" +generate_text_with_prompt(prompt, model, tokenizer) + +prompt = "Convert this Bliss sentence to an English sentence: present:They play merrily engage board by games cozy fireplace cozy evening cozy.\n" +generate_text_with_prompt(prompt, model, tokenizer) + +prompt = "Write a Bliss sentence of a greeting.\n" +generate_text_with_prompt(prompt, model, tokenizer) + +prompt = "Convert this sentence to a Bliss sentence: The Moon takes about one month to orbit Earth.\n" +generate_text_with_prompt(prompt, model, tokenizer) + +prompt = "Convert this sentence to a Bliss sentence: He studied hard because he wanted to go to medical school as he suffered from arthritis.\n" +generate_text_with_prompt(prompt, model, tokenizer) + +prompt = "Convert this Bliss sentence to an English sentence: past:he ride excitedly bike shiny new down road day next.\n" +generate_text_with_prompt(prompt, model, tokenizer) diff --git a/jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt b/jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt new file mode 100644 index 0000000..49c84ac --- /dev/null +++ b/jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt @@ -0,0 +1,130 @@ +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: pip in /home/cindyli/.local/lib/python3.11/site-packages (24.0+computecanada) +created virtual environment CPython3.11.5.final.0-64 in 6557ms + creator CPython3Posix(dest=/localscratch/cindyli.26600143.0/env, clear=False, no_vcs_ignore=False, global=False) + seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/home/cindyli/.local/share/virtualenv) + added seed packages: pip==23.3.2, setuptools==69.0.3, wheel==0.41.3 + activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator +/localscratch/spool/slurmd/job26600143/slurm_script: line 16: /localscratch/cindyli.26600143.0/.env/bin/activate: No such file or directory +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: pip in /home/cindyli/.local/lib/python3.11/site-packages (24.0+computecanada) + +Inactive Modules: + 1) libffi/3.3 2) sqlite/3.43.1 + +Due to MODULEPATH changes, the following have been reloaded: + 1) mii/1.1.2 2) python/3.11.5 + +The following have been reloaded with a version change: + 1) StdEnv/2020 => StdEnv/2023 5) libfabric/1.10.1 => libfabric/1.18.0 + 2) gcccore/.9.3.0 => gcccore/.12.3 6) openmpi/4.0.3 => openmpi/4.1.5 + 3) gentoo/2020 => gentoo/2023 7) ucx/1.8.0 => ucx/1.14.1 + 4) imkl/2020.1.217 => imkl/2023.2.0 + +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: torch in /home/cindyli/.local/lib/python3.11/site-packages (2.2.0+computecanada) +Requirement already satisfied: transformers==4.36.2 in /home/cindyli/.local/lib/python3.11/site-packages (4.36.2+computecanada) +Requirement already satisfied: peft==0.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (0.5.0+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from transformers==4.36.2) (3.12.2) +Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.20.2+computecanada) +Requirement already satisfied: numpy>=1.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (1.25.2+computecanada) +Requirement already satisfied: packaging>=20.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from transformers==4.36.2) (23.1+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (6.0.1+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2023.8.8+computecanada) +Requirement already satisfied: requests in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (2.31.0+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (0.4.1+computecanada) +Requirement already satisfied: tqdm>=4.27 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers==4.36.2) (4.66.1+computecanada) +Requirement already satisfied: psutil in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/ipykernel/2023b/lib/python3.11/site-packages (from peft==0.5.0) (5.9.5+computecanada) +Requirement already satisfied: accelerate in /home/cindyli/.local/lib/python3.11/site-packages (from peft==0.5.0) (0.25.0+computecanada) +Requirement already satisfied: typing-extensions>=4.8.0 in /home/cindyli/.local/lib/python3.11/site-packages (from torch) (4.9.0+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch) (3.2.1+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from torch) (3.1.3+computecanada) +Requirement already satisfied: fsspec in /home/cindyli/.local/lib/python3.11/site-packages (from torch) (2023.10.0+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->torch) (2.1.3+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests->transformers==4.36.2) (2023.7.22+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch) (1.3.0+computecanada) +=== Fine-tuning Llama2 from job 26600143 on nodes cdr2678. +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( + Loading checkpoint shards: 0%| | 0/2 [00:00 Date: Thu, 21 Mar 2024 14:47:07 -0400 Subject: [PATCH 10/20] feat: add script/job for evaluating generated sentences --- README.md | 7 ++ docs/Llama2FineTuning.md | 70 ++++++++++++++++++- .../finetune/eval_generated_sentence.py | 48 +++++++++++++ jobs/Llama2/finetune/finetune_7b_hf.py | 2 +- jobs/Llama2/finetune/job_eval_7b_hf.sh | 2 +- .../finetune/job_eval_generated_sentence.sh | 26 +++++++ jobs/Llama2/finetune/job_finetune_7b_hf.sh | 2 +- .../original_use/job_original_use_7b_hf.sh | 2 +- 8 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 jobs/Llama2/finetune/eval_generated_sentence.py create mode 100644 jobs/Llama2/finetune/job_eval_generated_sentence.sh diff --git a/README.md b/README.md index 49fd2a6..259577f 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,13 @@ Run the following command to lint all python scripts: We performed experiments with a number of existing models listed below to understand how useful they are in helping with generating new Bliss symbols etc. +### Llama2 + +Conclusion: useful + +See the [Llama2FineTuning.md](./docs/Llama2FineTuning.md) in the [documentation](./docs) folder for details +on how to fine tune, evaluation results and the conclusion about how useful it is. + ### StyleGAN3 Conclusion: not useful diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index af4646b..368cc82 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -33,7 +33,7 @@ git lfs fetch 6. Copy the content of [`requirements.txt`](https://github.com/facebookresearch/llama/blob/main/requirements.txt) for setting up the Llama2 models into a new file named `requirements-llama2.txt` in the "llama" directory. -## Use the Llama2 model +## Use the Original Llama2 Model In the [`jobs/original_use`](../jobs/Llama2/original_use) directory, there are two scripts: @@ -55,11 +55,11 @@ sbatch job_original_use_7b_hf.sh The result is written to the `llama2/original_use/result.txt`. -## Fine-tune the Llama2 model +## Fine-tune the Llama2 Model In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: -* bliss.json: The dataset that converts English text to the structure in the Bliss language +* bliss.json: The dataset that converts English text to the structure in the Conceptual Bliss * finetune_7b_hf.py: The script that fine-tunes the downloaded model * job_finetune_7b_hf.sh: The job script submitted to Cedar to run `finetune_7b_hf.py` @@ -72,6 +72,70 @@ cd scratch sbatch job_finetune_7b_hf.sh ``` +The fine-tuning script does: + +1. Create an instruction dataset using `bliss.json`. This dataset contains bi-directional conversion between +English and Conceptual Bliss. +2. Use the dataset to fine-tune the Llama2 model. See `finetune_7b_hf.py` about the fine-tuning parameters. +3. Evaluate the fine-tuned model by giving instructions fine-tuned for, along with a few sentences for language +conversion. + +Please note that due to the relatively small size of the dataset derived from bliss.json, the fine-tuning script +was run four times, adjusting the epoch number in the script from 1 to 4. As a result, 4 models were generated +corresponding to the different epoch counts. + +## Evaluate the Fine-tuned Model + +This section describes how to evaluate a fine-tuned model with instructions and input sentences. + +In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: + +* eval_7b_hf.py: The script that fine-tunes the downloaded model. Common variables to adjust: + * `model_dir`: The location of the model directory + * `instruction`: At the bottom of the script, define the instruction part in a prompt + * `input`: At the bottom of the script, define the sentence to be converted +* job_eval_7b_hf.sh: The job script submitted to Cedar to run `eval_7b_hf.py` + +FTP scripts above to the cedar cluster in the users `llama2/finetune` directory. Run the following command to +submit the job. + +``` +cp llama2/finetune/job_eval_7b_hf.sh scratch/. +cd scratch +sbatch job_eval_7b_hf.sh +``` + +## Evaluate the Generated Sentences from the Fine-tuned Model + +This section describes how to evaluat the generated sentences and compare them with original or expected sentences. +It evaluates the generated sentence in these aspects: + +* Semantic Coherence +* Novelty and Creativity +* Fluency and Readability + +In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: + +* eval_generated_sentence.py: The script that fine-tunes the downloaded model. Common variables to adjust: + * `sentence_orig`: The original sentence + * `sentence_expected`: The expected sentence + * `sentence_generated`: The sentence generated by the fine-tuned model +* job_eval_generated_sentence.sh: The job script submitted to Cedar to run `eval_generated_sentence.py` + +FTP scripts above to the cedar cluster in the users `llama2/finetune` directory. Run the following command to +submit the job. + +``` +cp llama2/finetune/job_eval_generated_sentence.sh scratch/. +cd scratch +sbatch job_eval_generated_sentence.sh +``` + +## Conclusion + +Although the fine-tuning uses a fairly small dataset, the fine-tuned model performs pretty well in converting English +and Conceptual Bliss sentence structure, especially with the two-epochs and three-epochs models. + ## References [Llama2 in the Facebook Research Github repository](https://github.com/facebookresearch/llama) diff --git a/jobs/Llama2/finetune/eval_generated_sentence.py b/jobs/Llama2/finetune/eval_generated_sentence.py new file mode 100644 index 0000000..b565dff --- /dev/null +++ b/jobs/Llama2/finetune/eval_generated_sentence.py @@ -0,0 +1,48 @@ +import spacy +from sentence_transformers import SentenceTransformer, util +from sklearn.feature_extraction.text import TfidfVectorizer +import numpy as np +import textstat + +# Sentence to convert +sentence_orig = "I had the pleasure of watching a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling." +# Expected sentence +sentence_expected = "past:I have pleasure of watching movie captivating that engage thoroughly my senses and emotions providing escape delightful into realm of storytelling cinematic." +# Model-generated sentence +sentence_generated = "past:I watch pleasure of movie captivating that engage thoroughly my senses emotions provide escape into realm storytelling cinematic." + +print(f"Original Sentence: \n{sentence_orig}") +print(f"Expected Sentence: \n{sentence_expected}") +print(f"Generated Sentence: \n{sentence_generated}") + +# Load the English tokenizer, tagger, parser, NER, and word vectors +nlp = spacy.load("en_core_web_sm") + +model = SentenceTransformer("all-MiniLM-L6-v2") + +# 1. Semantic Coherence +# Compute embedding for both lists +embeddings1 = model.encode(sentence_generated, convert_to_tensor=True) +embeddings2 = model.encode(sentence_expected, convert_to_tensor=True) + +# Compute cosine similarities +cosine_scores = util.pytorch_cos_sim(embeddings1, embeddings2) + +print("Cosine similarity between expected and generated sentences:", cosine_scores.item()) + +# 2. Novelty and Creativity +vectorizer = TfidfVectorizer() +X = vectorizer.fit_transform([sentence_orig] + [sentence_generated]) + +# Calculate similarity with the corpus +similarity = (X * X.T).A[-1][:-1] + +# Measure of novelty (1 - mean similarity) +novelty_score = 1 - np.mean(similarity) + +print("Novelty score by comparing with the original sentence:", novelty_score) + +# 3. Fluency and Readability +readability_score = textstat.flesch_reading_ease(sentence_generated) + +print("Readability Score:", readability_score) diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index c0fa05f..620c9a3 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -111,7 +111,7 @@ ################################################################################ # Maximum sequence length to use -max_seq_length = None +max_seq_length = 4096 # Pack multiple short examples in the same input sequence to increase efficiency packing = False diff --git a/jobs/Llama2/finetune/job_eval_7b_hf.sh b/jobs/Llama2/finetune/job_eval_7b_hf.sh index 201ae4d..069fb67 100644 --- a/jobs/Llama2/finetune/job_eval_7b_hf.sh +++ b/jobs/Llama2/finetune/job_eval_7b_hf.sh @@ -13,7 +13,7 @@ pip install --upgrade pip module load python/3.11.5 virtualenv --no-download $SLURM_TMPDIR/env -source $SLURM_TMPDIR/.env/bin/activate +source $SLURM_TMPDIR/env/bin/activate pip install --upgrade pip diff --git a/jobs/Llama2/finetune/job_eval_generated_sentence.sh b/jobs/Llama2/finetune/job_eval_generated_sentence.sh new file mode 100644 index 0000000..3313090 --- /dev/null +++ b/jobs/Llama2/finetune/job_eval_generated_sentence.sh @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --job-name=llama2-finetune-7b-hf +#SBATCH --time 2-00:00 +#SBATCH --nodes=1 +#SBATCH --gpus-per-node=v100l:1 +#SBATCH --mem=64G +#SBATCH --ntasks-per-node=4 +#SBATCH --cpus-per-task=4 +#SBATCH --account=def-whkchun +#SBATCH --output=%x.o%j + +pip install --upgrade pip +module load python/3.11.5 + +virtualenv --no-download $SLURM_TMPDIR/env +source $SLURM_TMPDIR/env/bin/activate + +pip install --upgrade pip + +module load StdEnv/2023 gcc/12.3 +pip install --no-index spacy sentence_transformers sklearn numpy +pip install textstat +python -m spacy download en_core_web_sm + +echo "=== Evaluate generated sentences from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." +python /home/cindyli/llama2/finetune/eval_generated_sentence.py diff --git a/jobs/Llama2/finetune/job_finetune_7b_hf.sh b/jobs/Llama2/finetune/job_finetune_7b_hf.sh index b12de61..549f796 100644 --- a/jobs/Llama2/finetune/job_finetune_7b_hf.sh +++ b/jobs/Llama2/finetune/job_finetune_7b_hf.sh @@ -13,7 +13,7 @@ pip install --upgrade pip module load python/3.11.5 virtualenv --no-download $SLURM_TMPDIR/env -source $SLURM_TMPDIR/.env/bin/activate +source $SLURM_TMPDIR/env/bin/activate pip install --upgrade pip diff --git a/jobs/Llama2/original_use/job_original_use_7b_hf.sh b/jobs/Llama2/original_use/job_original_use_7b_hf.sh index c2ff296..0c459bf 100644 --- a/jobs/Llama2/original_use/job_original_use_7b_hf.sh +++ b/jobs/Llama2/original_use/job_original_use_7b_hf.sh @@ -14,7 +14,7 @@ module load python/3.11.5 python -V virtualenv --no-download $SLURM_TMPDIR/env -source $SLURM_TMPDIR/.env/bin/activate +source $SLURM_TMPDIR/env/bin/activate pip install --upgrade pip From ea8f6dfad37cced2250ff6495d76019923ef9f06 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 22 Mar 2024 11:03:20 -0400 Subject: [PATCH 11/20] fix: add the result file for evaluating generated sentences --- .../7b-3epochs-eval-generated-sentence.txt | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 jobs/Llama2/finetune/results/7b-3epochs-eval-generated-sentence.txt diff --git a/jobs/Llama2/finetune/results/7b-3epochs-eval-generated-sentence.txt b/jobs/Llama2/finetune/results/7b-3epochs-eval-generated-sentence.txt new file mode 100644 index 0000000..f9fd0cd --- /dev/null +++ b/jobs/Llama2/finetune/results/7b-3epochs-eval-generated-sentence.txt @@ -0,0 +1,157 @@ +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: pip in /home/cindyli/.local/lib/python3.11/site-packages (24.0+computecanada) +created virtual environment CPython3.11.5.final.0-64 in 8775ms + creator CPython3Posix(dest=/localscratch/cindyli.27394030.0/env, clear=False, no_vcs_ignore=False, global=False) + seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/home/cindyli/.local/share/virtualenv) + added seed packages: pip==23.3.2, setuptools==69.0.3, wheel==0.41.3 + activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/avx2, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2020/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: pip in /localscratch/cindyli.27394030.0/env/lib/python3.11/site-packages (23.3.2) +Processing /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic/pip-24.0+computecanada-py3-none-any.whl +Installing collected packages: pip + Attempting uninstall: pip + Found existing installation: pip 23.3.2 + Uninstalling pip-23.3.2: + Successfully uninstalled pip-23.3.2 +Successfully installed pip-24.0+computecanada + +Inactive Modules: + 1) libffi/3.3 2) sqlite/3.43.1 + +Due to MODULEPATH changes, the following have been reloaded: + 1) mii/1.1.2 2) python/3.11.5 + +The following have been reloaded with a version change: + 1) StdEnv/2020 => StdEnv/2023 5) libfabric/1.10.1 => libfabric/1.18.0 + 2) gcccore/.9.3.0 => gcccore/.12.3 6) openmpi/4.0.3 => openmpi/4.1.5 + 3) gentoo/2020 => gentoo/2023 7) ucx/1.8.0 => ucx/1.14.1 + 4) imkl/2020.1.217 => imkl/2023.2.0 + +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: spacy in /home/cindyli/.local/lib/python3.11/site-packages (3.7.2+computecanada) +Requirement already satisfied: sentence_transformers in /home/cindyli/.local/lib/python3.11/site-packages (2.5.0+computecanada) +Requirement already satisfied: sklearn in /home/cindyli/.local/lib/python3.11/site-packages (0.0+computecanada) +Requirement already satisfied: numpy in /home/cindyli/.local/lib/python3.11/site-packages (1.25.2+computecanada) +Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (3.0.12+computecanada) +Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (1.0.5+computecanada) +Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (1.0.10+computecanada) +Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (2.0.8+computecanada) +Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (3.0.9+computecanada) +Requirement already satisfied: thinc<8.3.0,>=8.1.8 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (8.2.1+computecanada) +Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (1.1.2+computecanada) +Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (2.4.5+computecanada) +Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (2.0.10+computecanada) +Requirement already satisfied: weasel<0.4.0,>=0.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (0.3.4+computecanada) +Requirement already satisfied: typer<0.10.0,>=0.3.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (0.9.0+computecanada) +Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (6.4.0+computecanada) +Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (4.66.1+computecanada) +Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (2.31.0+computecanada) +Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (2.6.3+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (3.1.3+computecanada) +Requirement already satisfied: setuptools in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from spacy) (68.1.2) +Requirement already satisfied: packaging>=20.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from spacy) (23.1) +Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy) (3.3.0+computecanada) +Requirement already satisfied: transformers<5.0.0,>=4.32.0 in /home/cindyli/.local/lib/python3.11/site-packages (from sentence_transformers) (4.36.2+computecanada) +Requirement already satisfied: torch>=1.11.0 in /home/cindyli/.local/lib/python3.11/site-packages (from sentence_transformers) (2.2.0+computecanada) +Requirement already satisfied: scikit-learn in /home/cindyli/.local/lib/python3.11/site-packages (from sentence_transformers) (1.3.1+computecanada) +Requirement already satisfied: scipy in /home/cindyli/.local/lib/python3.11/site-packages (from sentence_transformers) (1.11.2+computecanada) +Requirement already satisfied: huggingface-hub>=0.15.1 in /home/cindyli/.local/lib/python3.11/site-packages (from sentence_transformers) (0.20.2+computecanada) +Requirement already satisfied: Pillow in /home/cindyli/.local/lib/python3.11/site-packages (from sentence_transformers) (10.2.0+computecanada) +Requirement already satisfied: filelock in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from huggingface-hub>=0.15.1->sentence_transformers) (3.12.2) +Requirement already satisfied: fsspec>=2023.5.0 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2023.10.0+computecanada) +Requirement already satisfied: pyyaml>=5.1 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub>=0.15.1->sentence_transformers) (6.0.1+computecanada) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from huggingface-hub>=0.15.1->sentence_transformers) (4.9.0+computecanada) +Requirement already satisfied: annotated-types>=0.4.0 in /home/cindyli/.local/lib/python3.11/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (0.6.0+computecanada) +Requirement already satisfied: pydantic-core==2.16.3 in /home/cindyli/.local/lib/python3.11/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (2.16.3+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2023.7.22+computecanada) +Requirement already satisfied: blis<0.8.0,>=0.7.8 in /home/cindyli/.local/lib/python3.11/site-packages (from thinc<8.3.0,>=8.1.8->spacy) (0.7.11+computecanada) +Requirement already satisfied: confection<1.0.0,>=0.0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from thinc<8.3.0,>=8.1.8->spacy) (0.1.4+computecanada) +Requirement already satisfied: sympy in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.11.0->sentence_transformers) (1.12+computecanada) +Requirement already satisfied: networkx in /home/cindyli/.local/lib/python3.11/site-packages (from torch>=1.11.0->sentence_transformers) (3.2.1+computecanada) +Requirement already satisfied: regex!=2019.12.17 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (2023.8.8+computecanada) +Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.15.0+computecanada) +Requirement already satisfied: safetensors>=0.3.1 in /home/cindyli/.local/lib/python3.11/site-packages (from transformers<5.0.0,>=4.32.0->sentence_transformers) (0.4.1+computecanada) +Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from typer<0.10.0,>=0.3.0->spacy) (8.1.7+computecanada) +Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from weasel<0.4.0,>=0.1.0->spacy) (0.16.0+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->spacy) (2.1.3+computecanada) +Requirement already satisfied: joblib>=1.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from scikit-learn->sentence_transformers) (1.3.2+computecanada) +Requirement already satisfied: threadpoolctl>=2.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from scikit-learn->sentence_transformers) (3.3.0+computecanada) +Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3.11/site-packages (from sympy->torch>=1.11.0->sentence_transformers) (1.3.0+computecanada) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Requirement already satisfied: textstat in /home/cindyli/.local/lib/python3.11/site-packages (0.7.3) +Requirement already satisfied: pyphen in /home/cindyli/.local/lib/python3.11/site-packages (from textstat) (0.14.0) +Defaulting to user installation because normal site-packages is not writeable +Looking in links: /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/x86-64-v3, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/gentoo2023/generic, /cvmfs/soft.computecanada.ca/custom/python/wheelhouse/generic +Collecting en-core-web-sm==3.7.1 + Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl (12.8 MB) + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 12.8/12.8 MB 2.2 MB/s eta 0:00:00 +Requirement already satisfied: spacy<3.8.0,>=3.7.2 in /home/cindyli/.local/lib/python3.11/site-packages (from en-core-web-sm==3.7.1) (3.7.2+computecanada) +Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.0.12+computecanada) +Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.0.5+computecanada) +Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.0.10+computecanada) +Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.8+computecanada) +Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.0.9+computecanada) +Requirement already satisfied: thinc<8.3.0,>=8.1.8 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (8.2.1+computecanada) +Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.1.2+computecanada) +Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.4.5+computecanada) +Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.0.10+computecanada) +Requirement already satisfied: weasel<0.4.0,>=0.1.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.3.4+computecanada) +Requirement already satisfied: typer<0.10.0,>=0.3.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.9.0+computecanada) +Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (6.4.0+computecanada) +Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (4.66.1+computecanada) +Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.31.0+computecanada) +Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.6.3+computecanada) +Requirement already satisfied: jinja2 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.1.3+computecanada) +Requirement already satisfied: setuptools in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (68.1.2) +Requirement already satisfied: packaging>=20.0 in /cvmfs/soft.computecanada.ca/easybuild/software/2023/x86-64-v3/Compiler/gcccore/python/3.11.5/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (23.1) +Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.3.0+computecanada) +Requirement already satisfied: numpy>=1.25 in /home/cindyli/.local/lib/python3.11/site-packages (from spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (1.25.2+computecanada) +Requirement already satisfied: annotated-types>=0.4.0 in /home/cindyli/.local/lib/python3.11/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.6.0+computecanada) +Requirement already satisfied: pydantic-core==2.16.3 in /home/cindyli/.local/lib/python3.11/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.16.3+computecanada) +Requirement already satisfied: typing-extensions>=4.6.1 in /home/cindyli/.local/lib/python3.11/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (4.9.0+computecanada) +Requirement already satisfied: charset-normalizer<4,>=2 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.2.0+computecanada) +Requirement already satisfied: idna<4,>=2.5 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (3.4+computecanada) +Requirement already satisfied: urllib3<3,>=1.21.1 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.1.0+computecanada) +Requirement already satisfied: certifi>=2017.4.17 in /home/cindyli/.local/lib/python3.11/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2023.7.22+computecanada) +Requirement already satisfied: blis<0.8.0,>=0.7.8 in /home/cindyli/.local/lib/python3.11/site-packages (from thinc<8.3.0,>=8.1.8->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.7.11+computecanada) +Requirement already satisfied: confection<1.0.0,>=0.0.1 in /home/cindyli/.local/lib/python3.11/site-packages (from thinc<8.3.0,>=8.1.8->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.1.4+computecanada) +Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/cindyli/.local/lib/python3.11/site-packages (from typer<0.10.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (8.1.7+computecanada) +Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in /home/cindyli/.local/lib/python3.11/site-packages (from weasel<0.4.0,>=0.1.0->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (0.16.0+computecanada) +Requirement already satisfied: MarkupSafe>=2.0 in /home/cindyli/.local/lib/python3.11/site-packages (from jinja2->spacy<3.8.0,>=3.7.2->en-core-web-sm==3.7.1) (2.1.3+computecanada) +Installing collected packages: en-core-web-sm +Successfully installed en-core-web-sm-3.7.1 +✔ Download and installation successful +You can now load the package via spacy.load('en_core_web_sm') +=== Evaluate generated sentences from job 27394030 on nodes cdr2497. +/home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + _torch_pytree._register_pytree_node( +Original Sentence: +I had the pleasure of watching a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling. +Expected Sentence: +past:I have pleasure of watching movie captivating that engage thoroughly my senses and emotions providing escape delightful into realm of storytelling cinematic. +Generated Sentence: +past:I watch pleasure of movie captivating that engage thoroughly my senses emotions provide escape into realm storytelling cinematic. + modules.json: 0%| | 0.00/349 [00:00 Date: Fri, 5 Apr 2024 10:58:02 -0400 Subject: [PATCH 12/20] doc: improve the documentaion for Llama2 fine-tuning --- docs/Llama2FineTuning.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index 368cc82..b1bd6d0 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -1,6 +1,14 @@ # Experiment with Llama2 model -The experiment leveraged the [7B parameter Llama2 model pretrained by Meta](https://huggingface.co/meta-llama/Llama-2-7b-hf), +The goal of this experiment is to evaluate how well a large language model (LLM) can learn the conversion +between English and Blissymbolics sentence structures. To leverage the LLM's knowledge of English, Blissymbolics +sentences are composed using English words while adhering to the grammatical and syntactical rules of Blissymbolics. +For instance, the English sentence "I slowly move towards the blue lake" would be expressed in Blissymbolics as +"present: I move slowly towards lake blue". Without delving into the linguistic intricacies of Blissymbolics, it is +essential to note that the language follows a specific ordering and structure to indicate verb tenses, as well as the +relationships between verbs and adverbs, and nouns and adjectives. + +The experiment uses the [7B parameter Llama2 model pretrained by Meta](https://huggingface.co/meta-llama/Llama-2-7b-hf), converted for the seamless use of the Hugging Face Transformers format. This model is choosen as a starting point because it requires less training time and GPU resources compared to its larger counterparts, while it potentially sacrifies some capability. Additionally, the Hugging Face Transformers format is selected because From e10b3d663890553ae320572e3cff34674a958f50 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 10 Apr 2024 13:51:34 -0400 Subject: [PATCH 13/20] fix: address review comments --- LICENSE | 2 +- docs/Llama2FineTuning.md | 46 +++++++++---------- jobs/Llama2/finetune/eval_7b_hf.py | 8 ++++ .../finetune/eval_generated_sentence.py | 8 ++++ jobs/Llama2/finetune/finetune_7b_hf.py | 8 ++++ jobs/Llama2/finetune/job_eval_7b_hf.sh | 9 ++++ .../finetune/job_eval_generated_sentence.sh | 9 ++++ jobs/Llama2/finetune/job_finetune_7b_hf.sh | 9 ++++ .../original_use/job_original_use_7b_hf.sh | 9 ++++ .../Llama2/original_use/original_use_7b_hf.py | 8 ++++ .../ctb-styleGAN2AdaPytorchGenerateBatch.sh | 4 +- .../def-styleGAN2AdaPytorchGenerateBatch.sh | 4 +- .../def-styleGAN2AdaPytorchTrainBatch.sh | 3 +- .../def-styleGan2AdaPytorchDataSetupBatch.sh | 8 +--- utils/scale_down_images.py | 2 +- 15 files changed, 97 insertions(+), 40 deletions(-) diff --git a/LICENSE b/LICENSE index a1a39d5..3e33e83 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2023, Inclusive Design Institute +Copyright (c) 2023-2024, Inclusive Design Institute Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index b1bd6d0..c6bc589 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -27,7 +27,8 @@ to request the access to its Llama2 model; mkdir llama2 cd llama2 -// Load git-lfs first for downloading via Git large file storage +# Load git-lfs first for downloading via Git large file storage +module load StdEnv/2020 module load git-lfs/3.3.0 git lfs install @@ -43,7 +44,7 @@ for setting up the Llama2 models into a new file named `requirements-llama2.txt` ## Use the Original Llama2 Model -In the [`jobs/original_use`](../jobs/Llama2/original_use) directory, there are two scripts: +In the [`jobs/Llama2/original_use`](../jobs/Llama2/original_use) directory, there are two scripts: * original_use_7b_hf.py: The script that loads the downloaded model and tokenizer to perform text generation, word predictions and making inferences @@ -52,8 +53,8 @@ word predictions and making inferences Note that the job script must be copied to the user's `scratch` directory and is submitted from there using the `sbatch` command. -FTP scripts above to the cedar cluster in the users `llama2/original_use` directory. Run the following command to -submit the job. +Use FTP to transfer the above scripts to the cedar cluster in the users `llama2/original_use` directory. Run +the following command to submit the job. ``` cp llama2/original_use/job_original_use_7b_hf.sh scratch/. @@ -65,14 +66,14 @@ The result is written to the `llama2/original_use/result.txt`. ## Fine-tune the Llama2 Model -In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: +In the [`jobs/Llama2/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: * bliss.json: The dataset that converts English text to the structure in the Conceptual Bliss * finetune_7b_hf.py: The script that fine-tunes the downloaded model * job_finetune_7b_hf.sh: The job script submitted to Cedar to run `finetune_7b_hf.py` -FTP scripts above to the cedar cluster in the users `llama2/finetune` directory. Run the following command to -submit the job. +Use FTP to transfer the above scripts to the cedar cluster in the users `llama2/finetune` directory. Run +the following command to submit the job. ``` cp llama2/finetune/job_finetune_7b_hf.sh scratch/. @@ -80,13 +81,12 @@ cd scratch sbatch job_finetune_7b_hf.sh ``` -The fine-tuning script does: +The fine-tuning script: -1. Create an instruction dataset using `bliss.json`. This dataset contains bi-directional conversion between +1. Creates an instruction dataset using `bliss.json`. This dataset contains bi-directional conversion between English and Conceptual Bliss. -2. Use the dataset to fine-tune the Llama2 model. See `finetune_7b_hf.py` about the fine-tuning parameters. -3. Evaluate the fine-tuned model by giving instructions fine-tuned for, along with a few sentences for language -conversion. +2. Uses the dataset to fine-tune the Llama2 model. See `finetune_7b_hf.py` about the fine-tuning parameters. +3. Evaluates the fine-tuned model by testing a few sentence conversions between the English and the Bliss languages. Please note that due to the relatively small size of the dataset derived from bliss.json, the fine-tuning script was run four times, adjusting the epoch number in the script from 1 to 4. As a result, 4 models were generated @@ -96,7 +96,7 @@ corresponding to the different epoch counts. This section describes how to evaluate a fine-tuned model with instructions and input sentences. -In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: +In the [`jobs/Llama2/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: * eval_7b_hf.py: The script that fine-tunes the downloaded model. Common variables to adjust: * `model_dir`: The location of the model directory @@ -104,8 +104,8 @@ In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scr * `input`: At the bottom of the script, define the sentence to be converted * job_eval_7b_hf.sh: The job script submitted to Cedar to run `eval_7b_hf.py` -FTP scripts above to the cedar cluster in the users `llama2/finetune` directory. Run the following command to -submit the job. +Use FTP to transfer the above scripts to the cedar cluster in the users `llama2/finetune` directory. Run +the following command to submit the job. ``` cp llama2/finetune/job_eval_7b_hf.sh scratch/. @@ -115,14 +115,14 @@ sbatch job_eval_7b_hf.sh ## Evaluate the Generated Sentences from the Fine-tuned Model -This section describes how to evaluat the generated sentences and compare them with original or expected sentences. +This section describes how to evaluate the generated sentences and compare them with original or expected sentences. It evaluates the generated sentence in these aspects: * Semantic Coherence * Novelty and Creativity * Fluency and Readability -In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: +In the [`jobs/Llama2/finetune`](../jobs/Llama2/finetune) directory, there are these scripts: * eval_generated_sentence.py: The script that fine-tunes the downloaded model. Common variables to adjust: * `sentence_orig`: The original sentence @@ -130,8 +130,8 @@ In the [`jobs/finetune`](../jobs/Llama2/finetune) directory, there are these scr * `sentence_generated`: The sentence generated by the fine-tuned model * job_eval_generated_sentence.sh: The job script submitted to Cedar to run `eval_generated_sentence.py` -FTP scripts above to the cedar cluster in the users `llama2/finetune` directory. Run the following command to -submit the job. +Use FTP to transfer the above scripts to the cedar cluster in the users `llama2/finetune` directory. Run +the following command to submit the job. ``` cp llama2/finetune/job_eval_generated_sentence.sh scratch/. @@ -146,7 +146,7 @@ and Conceptual Bliss sentence structure, especially with the two-epochs and thre ## References -[Llama2 in the Facebook Research Github repository](https://github.com/facebookresearch/llama) -[Llama2 fine-tune, inference examples](https://github.com/facebookresearch/llama-recipes) -[Llama2 on Hugging Face](https://huggingface.co/docs/transformers/model_doc/llama2) -[Use Hugging Face Models on Cedar Clusters](https://docs.alliancecan.ca/wiki/Huggingface) +* [Llama2 in the Facebook Research Github repository](https://github.com/facebookresearch/llama) +* [Llama2 fine-tune, inference examples](https://github.com/facebookresearch/llama-recipes) +* [Llama2 on Hugging Face](https://huggingface.co/docs/transformers/model_doc/llama2) +* [Use Hugging Face Models on Cedar Clusters](https://docs.alliancecan.ca/wiki/Huggingface) diff --git a/jobs/Llama2/finetune/eval_7b_hf.py b/jobs/Llama2/finetune/eval_7b_hf.py index 64ecd88..d9f68c2 100644 --- a/jobs/Llama2/finetune/eval_7b_hf.py +++ b/jobs/Llama2/finetune/eval_7b_hf.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + import torch from peft import AutoPeftModelForCausalLM from transformers import AutoTokenizer diff --git a/jobs/Llama2/finetune/eval_generated_sentence.py b/jobs/Llama2/finetune/eval_generated_sentence.py index b565dff..81bcada 100644 --- a/jobs/Llama2/finetune/eval_generated_sentence.py +++ b/jobs/Llama2/finetune/eval_generated_sentence.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + import spacy from sentence_transformers import SentenceTransformer, util from sklearn.feature_extraction.text import TfidfVectorizer diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 620c9a3..36b1ba5 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + import torch from datasets import load_dataset, concatenate_datasets from transformers import ( diff --git a/jobs/Llama2/finetune/job_eval_7b_hf.sh b/jobs/Llama2/finetune/job_eval_7b_hf.sh index 069fb67..b3fdfd1 100644 --- a/jobs/Llama2/finetune/job_eval_7b_hf.sh +++ b/jobs/Llama2/finetune/job_eval_7b_hf.sh @@ -1,4 +1,13 @@ #!/bin/bash + +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + #SBATCH --job-name=llama2-finetune-7b-hf #SBATCH --time 2-00:00 #SBATCH --nodes=1 diff --git a/jobs/Llama2/finetune/job_eval_generated_sentence.sh b/jobs/Llama2/finetune/job_eval_generated_sentence.sh index 3313090..5a7c078 100644 --- a/jobs/Llama2/finetune/job_eval_generated_sentence.sh +++ b/jobs/Llama2/finetune/job_eval_generated_sentence.sh @@ -1,4 +1,13 @@ #!/bin/bash + +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + #SBATCH --job-name=llama2-finetune-7b-hf #SBATCH --time 2-00:00 #SBATCH --nodes=1 diff --git a/jobs/Llama2/finetune/job_finetune_7b_hf.sh b/jobs/Llama2/finetune/job_finetune_7b_hf.sh index 549f796..5a889a0 100644 --- a/jobs/Llama2/finetune/job_finetune_7b_hf.sh +++ b/jobs/Llama2/finetune/job_finetune_7b_hf.sh @@ -1,4 +1,13 @@ #!/bin/bash + +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + #SBATCH --job-name=llama2-finetune-7b-hf #SBATCH --time 2-00:00 #SBATCH --nodes=1 diff --git a/jobs/Llama2/original_use/job_original_use_7b_hf.sh b/jobs/Llama2/original_use/job_original_use_7b_hf.sh index 0c459bf..3f40041 100644 --- a/jobs/Llama2/original_use/job_original_use_7b_hf.sh +++ b/jobs/Llama2/original_use/job_original_use_7b_hf.sh @@ -1,4 +1,13 @@ #!/bin/bash + +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + #SBATCH --job-name=llama2-orig-use-7b-hf #SBATCH --time 10-00:00 #SBATCH --nodes=1 diff --git a/jobs/Llama2/original_use/original_use_7b_hf.py b/jobs/Llama2/original_use/original_use_7b_hf.py index 0ad960d..7b11b9e 100644 --- a/jobs/Llama2/original_use/original_use_7b_hf.py +++ b/jobs/Llama2/original_use/original_use_7b_hf.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline model_dir = "/home/cindyli/llama2/Llama-2-7b-hf" diff --git a/jobs/stylegan2-ada/ctb-styleGAN2AdaPytorchGenerateBatch.sh b/jobs/stylegan2-ada/ctb-styleGAN2AdaPytorchGenerateBatch.sh index 8238982..072335e 100755 --- a/jobs/stylegan2-ada/ctb-styleGAN2AdaPytorchGenerateBatch.sh +++ b/jobs/stylegan2-ada/ctb-styleGAN2AdaPytorchGenerateBatch.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, Inclusive Design Institute +# Copyright (c) 2023-2024, Inclusive Design Institute # # Licensed under the BSD 3-Clause License. You may not use this file except # in compliance with this License. @@ -39,5 +39,3 @@ mkdir -p "$OUTPUT_DIR" # Generate... python ~/BlissStyleGAN/StyleGAN2/stylegan2-ada-pytorch/generate.py --outdir="$OUTPUT_DIR" --trunc=0.5 --seeds=200,330,400 --network="$MODEL_FILE" - - diff --git a/jobs/stylegan2-ada/def-styleGAN2AdaPytorchGenerateBatch.sh b/jobs/stylegan2-ada/def-styleGAN2AdaPytorchGenerateBatch.sh index d3ea406..ac2618b 100755 --- a/jobs/stylegan2-ada/def-styleGAN2AdaPytorchGenerateBatch.sh +++ b/jobs/stylegan2-ada/def-styleGAN2AdaPytorchGenerateBatch.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, Inclusive Design Institute +# Copyright (c) 2023-2024, Inclusive Design Institute # # Licensed under the BSD 3-Clause License. You may not use this file except # in compliance with this License. @@ -41,5 +41,3 @@ mkdir -p "$OUTPUT_DIR" # # This third command is resuming for another 12 hours, using latest model python ~/BlissStyleGAN/StyleGAN2/stylegan2-ada-pytorch/generate.py --outdir="$OUTPUT_DIR" --trunc=0.5 --seeds=600-605 --network="$MODEL_FILE" - - diff --git a/jobs/stylegan2-ada/def-styleGAN2AdaPytorchTrainBatch.sh b/jobs/stylegan2-ada/def-styleGAN2AdaPytorchTrainBatch.sh index 51f952e..c40907f 100755 --- a/jobs/stylegan2-ada/def-styleGAN2AdaPytorchTrainBatch.sh +++ b/jobs/stylegan2-ada/def-styleGAN2AdaPytorchTrainBatch.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, Inclusive Design Institute +# Copyright (c) 2023-2024, Inclusive Design Institute # # Licensed under the BSD 3-Clause License. You may not use this file except # in compliance with this License. @@ -58,4 +58,3 @@ python ~/BlissStyleGAN/StyleGAN2/stylegan2-ada-pytorch/train.py --outdir="$OUTPU # This third command is resuming for another 15 hours, using latest model. # Again, the actual values here may differ for different groups of runs. # python ~/BlissStyleGAN/StyleGAN2/stylegan2-ada-pytorch/train.py --outdir="$OUTPUT_DIR" --data="$DATA_DIR" --snap=10 --resume="$OUTPUT_DIR/00001-preppedBlissSingleCharGrey-auto1-resumecustom/network-snapshot-000440.pkl" - diff --git a/jobs/stylegan2-ada/def-styleGan2AdaPytorchDataSetupBatch.sh b/jobs/stylegan2-ada/def-styleGan2AdaPytorchDataSetupBatch.sh index 4e2a221..515009f 100755 --- a/jobs/stylegan2-ada/def-styleGan2AdaPytorchDataSetupBatch.sh +++ b/jobs/stylegan2-ada/def-styleGan2AdaPytorchDataSetupBatch.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, Inclusive Design Institute +# Copyright (c) 2023-2024, Inclusive Design Institute # # Licensed under the BSD 3-Clause License. You may not use this file except # in compliance with this License. @@ -59,9 +59,3 @@ else echo "dataset_tool.py failed with exit status $STATUS" fi echo Done! - - - - - - diff --git a/utils/scale_down_images.py b/utils/scale_down_images.py index ca53e0b..7fd49a0 100644 --- a/utils/scale_down_images.py +++ b/utils/scale_down_images.py @@ -3,7 +3,7 @@ from PIL import Image """ -Copyright (c) 2023, Inclusive Design Institute +Copyright (c) 2023-2024, Inclusive Design Institute Licensed under the BSD 3-Clause License. You may not use this file except in compliance with this License. From b3083b1ad342f8dd3c90a5d4865a97e6671619d3 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 10 Apr 2024 14:51:40 -0400 Subject: [PATCH 14/20] fix: address more review comments --- docs/Llama2FineTuning.md | 2 ++ jobs/Llama2/finetune/eval_7b_hf.py | 8 +++---- jobs/Llama2/finetune/finetune_7b_hf.py | 24 +++++++++++-------- jobs/Llama2/finetune/job_eval_7b_hf.sh | 2 +- .../finetune/job_eval_generated_sentence.sh | 2 +- jobs/Llama2/finetune/job_finetune_7b_hf.sh | 4 ++-- .../results/7b-3epochs-other-prompts.txt | 20 +++++++++------- .../original_use/job_original_use_7b_hf.sh | 4 ++-- .../Llama2/original_use/original_use_7b_hf.py | 4 ++-- jobs/stylegan3/job_stylegan3-t_gen_images.sh | 4 ++-- jobs/stylegan3/job_stylegan3.sh | 4 ++-- 11 files changed, 43 insertions(+), 35 deletions(-) diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index c6bc589..647d785 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -150,3 +150,5 @@ and Conceptual Bliss sentence structure, especially with the two-epochs and thre * [Llama2 fine-tune, inference examples](https://github.com/facebookresearch/llama-recipes) * [Llama2 on Hugging Face](https://huggingface.co/docs/transformers/model_doc/llama2) * [Use Hugging Face Models on Cedar Clusters](https://docs.alliancecan.ca/wiki/Huggingface) +* [Running Jobs](https://docs.alliancecan.ca/wiki/Using_GPUs_with_Slurm) +* [Request GPUs with Slurm](https://docs.alliancecan.ca/wiki/Using_GPUs_with_Slurm) diff --git a/jobs/Llama2/finetune/eval_7b_hf.py b/jobs/Llama2/finetune/eval_7b_hf.py index d9f68c2..7d2fc8c 100644 --- a/jobs/Llama2/finetune/eval_7b_hf.py +++ b/jobs/Llama2/finetune/eval_7b_hf.py @@ -10,7 +10,7 @@ from peft import AutoPeftModelForCausalLM from transformers import AutoTokenizer -model_dir = "/home/cindyli/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf-3epochs/checkpoint-975" +model_dir = "~/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf-3epochs/checkpoint-975" # load base LLM model and tokenizer model = AutoPeftModelForCausalLM.from_pretrained( @@ -33,14 +33,14 @@ def generate_text_with_instruction(instruction, input, model, tokenizer, tempera print(f"Instruction: {instruction}\n") print(f"Prompt: {input}\n") outputs = model.generate(input_ids=input_ids, max_new_tokens=100, do_sample=True, top_p=0.9, temperature=temperature) - print(f"Generated instruction (temprature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") + print(f"Generated instruction (temperature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") def generate_text_with_prompt(prompt, model, tokenizer, temperature=0.7): input_ids = tokenizer(prompt, return_tensors="pt", truncation=True).input_ids.cuda() print(f"Prompt: {prompt}\n") outputs = model.generate(input_ids=input_ids, max_new_tokens=100, do_sample=True, top_p=0.9, temperature=temperature) - print(f"Generated instruction (temprature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") + print(f"Generated instruction (temperature {temperature}): {tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0][len(prompt):]}\n\n") # Test with exactly same instructions used in fine-tuning @@ -61,7 +61,7 @@ def generate_text_with_prompt(prompt, model, tokenizer, temperature=0.7): input = "future:month next, I embark on an journey exciting to explore the cultures vibrant and landscapes breathtaking of Southeast Asia." generate_text_with_instruction(instruction, input, model, tokenizer) -# Test with random propmts +# Test with random prompts # Two prompts below is copied from the dataset prompt = "Convert this sentence to a Bliss sentence: He rode his skateboard at the skate park yesterday.\n" generate_text_with_prompt(prompt, model, tokenizer) diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 36b1ba5..896585e 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -16,17 +16,16 @@ pipeline ) from peft import LoraConfig -# from peft import PeftModel from trl import SFTTrainer # The local directory with the model and the tokenizer -model_dir = "/home/cindyli/projects/ctb-whkchun/s2_bliss_LLMs/Llama-2-7b-hf" +model_dir = "~/projects/ctb-whkchun/s2_bliss_LLMs/Llama-2-7b-hf" # The instruction dataset to use -dataset_name = "/home/cindyli/llama2/finetune/bliss.json" +dataset_name = "~/llama2/finetune/bliss.json" # Output directory where the model checkpoints will be stored -output_dir = "/home/cindyli/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf" +output_dir = "~/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf" # Fine-tuned model name new_model = "llama-2-7b-hf-bliss" @@ -127,12 +126,17 @@ # Load the entire model on the GPU 0 device_map = {"": 0} +# Static instructions +instructions_map = { + "EnglishToBliss": "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n", + "BlissToEnglish": "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" +} # Create a formatted prompt template for an entry in the dataset # The "direction" parameter accepts either "EnglishToBliss" or "BlissToEnglish" -def format_prompt(sample, direction="EnglishToBliss"): +def format_prompt(sample, instructions_map, direction="EnglishToBliss"): # Initialize static strings for the prompt template - instruction = f"### Instruction: \nConvert the input {'English' if direction=='EnglishToBliss' else 'Bliss'} sentence to a {'Bliss' if direction=='EnglishToBliss' else 'English'} sentence.\n\n" + instruction = instructions_map[direction] input_key = "### Input:\n" response_key = "### Response:\n" @@ -186,8 +190,8 @@ def format_prompt(sample, direction="EnglishToBliss"): print(f"Column names are: {orig_dataset.column_names}") # Create two datasets for English to Bliss and Bliss to English -english_to_bliss_dataset = orig_dataset.map(lambda x: format_prompt(x, direction="EnglishToBliss")) -bliss_to_english_dataset = orig_dataset.map(lambda x: format_prompt(x, direction="BlissToEnglish")) +english_to_bliss_dataset = orig_dataset.map(lambda x: format_prompt(x, instructions_map, direction="EnglishToBliss")) +bliss_to_english_dataset = orig_dataset.map(lambda x: format_prompt(x, instructions_map, direction="BlissToEnglish")) # Combine the two datasets dataset = concatenate_datasets([english_to_bliss_dataset, bliss_to_english_dataset]) @@ -275,7 +279,7 @@ def predict_words(prompt, model, tokenizer): print("1. Inference\n") -instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" +instruction = instructions_map["EnglishToBliss"] input = "I am a programmer." generate_text(instruction, input, model, tokenizer) @@ -285,7 +289,7 @@ def predict_words(prompt, model, tokenizer): input = "I had the pleasure of watching a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling." generate_text(instruction, input, model, tokenizer) -instruction = "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" +instruction = instructions_map["BlissToEnglish"] input = "past:The girl run in the park." generate_text(instruction, input, model, tokenizer) diff --git a/jobs/Llama2/finetune/job_eval_7b_hf.sh b/jobs/Llama2/finetune/job_eval_7b_hf.sh index b3fdfd1..a19816b 100644 --- a/jobs/Llama2/finetune/job_eval_7b_hf.sh +++ b/jobs/Llama2/finetune/job_eval_7b_hf.sh @@ -30,4 +30,4 @@ module load StdEnv/2023 rust/1.70.0 arrow/14.0.1 gcc/12.3 pip install --no-index torch transformers==4.36.2 peft==0.5.0 echo "=== Fine-tuning Llama2 from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." -python /home/cindyli/llama2/finetune/eval_7b_hf.py +python ~/llama2/finetune/eval_7b_hf.py diff --git a/jobs/Llama2/finetune/job_eval_generated_sentence.sh b/jobs/Llama2/finetune/job_eval_generated_sentence.sh index 5a7c078..bc94d8f 100644 --- a/jobs/Llama2/finetune/job_eval_generated_sentence.sh +++ b/jobs/Llama2/finetune/job_eval_generated_sentence.sh @@ -32,4 +32,4 @@ pip install textstat python -m spacy download en_core_web_sm echo "=== Evaluate generated sentences from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." -python /home/cindyli/llama2/finetune/eval_generated_sentence.py +python ~/llama2/finetune/eval_generated_sentence.py diff --git a/jobs/Llama2/finetune/job_finetune_7b_hf.sh b/jobs/Llama2/finetune/job_finetune_7b_hf.sh index 5a889a0..18d8477 100644 --- a/jobs/Llama2/finetune/job_finetune_7b_hf.sh +++ b/jobs/Llama2/finetune/job_finetune_7b_hf.sh @@ -29,7 +29,7 @@ pip install --upgrade pip module load StdEnv/2023 rust/1.70.0 arrow/14.0.1 gcc/12.3 pip install --no-index transformers==4.36.2 accelerate==0.25.0 peft==0.5.0 bitsandbytes==0.42.0 tensorboard pip install datasets==2.17.0 trl -pip install -r /home/cindyli/llama2/requirements-llama2.txt +pip install -r ~/llama2/requirements-llama2.txt echo "=== Fine-tuning Llama2 from job $SLURM_JOB_ID on nodes $SLURM_JOB_NODELIST." -python /home/cindyli/llama2/finetune/finetune_7b_hf.py +python ~/llama2/finetune/finetune_7b_hf.py diff --git a/jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt b/jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt index 49c84ac..ee928ad 100644 --- a/jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt +++ b/jobs/Llama2/finetune/results/7b-3epochs-other-prompts.txt @@ -56,10 +56,14 @@ Requirement already satisfied: mpmath>=0.19 in /home/cindyli/.local/lib/python3. _torch_pytree._register_pytree_node( /home/cindyli/.local/lib/python3.11/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. _torch_pytree._register_pytree_node( - Loading checkpoint shards: 0%| | 0/2 [00:00 Date: Wed, 10 Apr 2024 15:08:17 -0400 Subject: [PATCH 15/20] fix: linted --- jobs/Llama2/finetune/finetune_7b_hf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 896585e..8820792 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -132,6 +132,7 @@ "BlissToEnglish": "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" } + # Create a formatted prompt template for an entry in the dataset # The "direction" parameter accepts either "EnglishToBliss" or "BlissToEnglish" def format_prompt(sample, instructions_map, direction="EnglishToBliss"): From 95d6aad7cf925e5fe025d6ac22a6fbc174f6f6a9 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 10 Apr 2024 15:20:15 -0400 Subject: [PATCH 16/20] fix: improve the documentation for the Llama2 fine tuning --- docs/Llama2FineTuning.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index 647d785..49c4438 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -14,6 +14,10 @@ point because it requires less training time and GPU resources compared to its l potentially sacrifies some capability. Additionally, the Hugging Face Transformers format is selected because of its extensive community support and standardized APIs. +This experiment is performed using Cedar clusters provided by [Digital Research Alliance of Canada](https://alliancecan.ca/en). +Refer to [its technical documentation](https://docs.alliancecan.ca/wiki/Technical_documentation) regarding the content of +job scripts and job submission steps described below. + ## Download Llama-2-7b-hf to Cedar 1. Request access to Llama2 models on [the Meta website](https://llama.meta.com/llama-downloads/); From 9a7d82dded43db966836e4fb8bca65baba78694b Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 17 Apr 2024 11:38:02 -0400 Subject: [PATCH 17/20] fix: improve the documentation for Llama2 fine-tuning --- docs/Llama2FineTuning.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index 49c4438..85b693d 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -15,7 +15,7 @@ potentially sacrifies some capability. Additionally, the Hugging Face Transforme of its extensive community support and standardized APIs. This experiment is performed using Cedar clusters provided by [Digital Research Alliance of Canada](https://alliancecan.ca/en). -Refer to [its technical documentation](https://docs.alliancecan.ca/wiki/Technical_documentation) regarding the content of +See [its technical documentation](https://docs.alliancecan.ca/wiki/Technical_documentation) regarding the content of job scripts and job submission steps described below. ## Download Llama-2-7b-hf to Cedar @@ -143,6 +143,28 @@ cd scratch sbatch job_eval_generated_sentence.sh ``` +## Future Improvements + +## Future Improvements + +1. **Diversified Dataset Expansion**: Currently, the `bliss.json` dataset consists of 967 pairs of conversions +between English and Bliss, focusing on specific ordering and sentence structures. To enhance the model's versatility, +a key improvement is to enrich the dataset with a wider variety of sentence types and structures. + +2. **Comprehensive Model Evaluation**: The evaluation of the fine-tuned model is not comprehensive. While individual +converted sentences are assessed, there's a need for a more thorough evaluation method. This includes comparing the +expected and actual converted results using a percentage of the dataset, and assessing for underfitting or overfitting. +Considering the fine-tuning runs from 1 to 4 epochs on a small dataset, overfitting risks may increase with more +epochs, which reqires a robust evaluation process. + +3. **Understanding Bliss Language**: The current fine-tuned model effectively responds to two fixed instructions, +converting between English and Bliss. However, it lacks a deep understanding of the Bliss language itself. The next +step involves fine-tuning a model that comprehends broader queries in Bliss, going beyond instructional conversion +tasks. Tests show that while the model performs well in converting Bliss to English, likely because of its extensive +knowledge of English. However, its performance in the reverse direction is not ideal. This difference suggests a need +for additional fine-tuning, potentially by enhancing the model's understanding of the unique linguistic features of +Bliss. + ## Conclusion Although the fine-tuning uses a fairly small dataset, the fine-tuned model performs pretty well in converting English From 217c50f2f54c8ffef0c623c6d1a1faf5486d8eae Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 17 Apr 2024 11:40:45 -0400 Subject: [PATCH 18/20] fix: remove duplicates --- docs/Llama2FineTuning.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/Llama2FineTuning.md b/docs/Llama2FineTuning.md index 85b693d..40b6766 100644 --- a/docs/Llama2FineTuning.md +++ b/docs/Llama2FineTuning.md @@ -145,8 +145,6 @@ sbatch job_eval_generated_sentence.sh ## Future Improvements -## Future Improvements - 1. **Diversified Dataset Expansion**: Currently, the `bliss.json` dataset consists of 967 pairs of conversions between English and Bliss, focusing on specific ordering and sentence structures. To enhance the model's versatility, a key improvement is to enrich the dataset with a wider variety of sentence types and structures. From ec7968e8ab3840538dc4cb144cc1411366aa5f7a Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 22 Apr 2024 13:27:33 -0400 Subject: [PATCH 19/20] fix: address review comments --- jobs/Llama2/finetune/eval_7b_hf.py | 15 ++++++++------- jobs/Llama2/finetune/finetune_7b_hf.py | 14 +++++--------- jobs/Llama2/finetune/shared_data.py | 13 +++++++++++++ jobs/stylegan3/job_stylegan3-t_gen_images.sh | 9 +++++++++ jobs/stylegan3/job_stylegan3.sh | 9 +++++++++ 5 files changed, 44 insertions(+), 16 deletions(-) create mode 100644 jobs/Llama2/finetune/shared_data.py diff --git a/jobs/Llama2/finetune/eval_7b_hf.py b/jobs/Llama2/finetune/eval_7b_hf.py index 7d2fc8c..8045c43 100644 --- a/jobs/Llama2/finetune/eval_7b_hf.py +++ b/jobs/Llama2/finetune/eval_7b_hf.py @@ -6,11 +6,13 @@ # You may obtain a copy of the BSD 3-Clause License at # https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE +import os import torch from peft import AutoPeftModelForCausalLM from transformers import AutoTokenizer +from shared_data import instructions_map -model_dir = "~/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf-3epochs/checkpoint-975" +model_dir = os.path.expanduser("~") + "/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf-3epochs/checkpoint-975" # load base LLM model and tokenizer model = AutoPeftModelForCausalLM.from_pretrained( @@ -46,20 +48,19 @@ def generate_text_with_prompt(prompt, model, tokenizer, temperature=0.7): # Test with exactly same instructions used in fine-tuning instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" input = "I am a programmer." -generate_text_with_instruction(instruction, input, model, tokenizer) +generate_text_with_instruction(instructions_map["EnglishToBliss"], input, model, tokenizer) input = "Joe will explore the picturesque landscapes of a charming countryside village tomorrow." -generate_text_with_instruction(instruction, input, model, tokenizer) +generate_text_with_instruction(instructions_map["EnglishToBliss"], input, model, tokenizer) input = "I had the pleasure of watching a captivating movie that thoroughly engaged my senses and emotions, providing a delightful escape into the realm of cinematic storytelling." -generate_text_with_instruction(instruction, input, model, tokenizer) +generate_text_with_instruction(instructions_map["EnglishToBliss"], input, model, tokenizer) -instruction = "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" input = "past:The girl run in the park." -generate_text_with_instruction(instruction, input, model, tokenizer) +generate_text_with_instruction(instructions_map["BlissToEnglish"], input, model, tokenizer) input = "future:month next, I embark on an journey exciting to explore the cultures vibrant and landscapes breathtaking of Southeast Asia." -generate_text_with_instruction(instruction, input, model, tokenizer) +generate_text_with_instruction(instructions_map["BlissToEnglish"], input, model, tokenizer) # Test with random prompts # Two prompts below is copied from the dataset diff --git a/jobs/Llama2/finetune/finetune_7b_hf.py b/jobs/Llama2/finetune/finetune_7b_hf.py index 8820792..4da65fb 100644 --- a/jobs/Llama2/finetune/finetune_7b_hf.py +++ b/jobs/Llama2/finetune/finetune_7b_hf.py @@ -6,6 +6,7 @@ # You may obtain a copy of the BSD 3-Clause License at # https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE +import os import torch from datasets import load_dataset, concatenate_datasets from transformers import ( @@ -17,15 +18,16 @@ ) from peft import LoraConfig from trl import SFTTrainer +from shared_data import instructions_map # The local directory with the model and the tokenizer -model_dir = "~/projects/ctb-whkchun/s2_bliss_LLMs/Llama-2-7b-hf" +model_dir = os.path.expanduser("~") + "/projects/ctb-whkchun/s2_bliss_LLMs/Llama-2-7b-hf" # The instruction dataset to use -dataset_name = "~/llama2/finetune/bliss.json" +dataset_name = os.path.expanduser("~") + "/llama2/finetune/bliss.json" # Output directory where the model checkpoints will be stored -output_dir = "~/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf" +output_dir = os.path.expanduser("~") + "/projects/ctb-whkchun/s2_bliss/results-finetune-7b-hf" # Fine-tuned model name new_model = "llama-2-7b-hf-bliss" @@ -126,12 +128,6 @@ # Load the entire model on the GPU 0 device_map = {"": 0} -# Static instructions -instructions_map = { - "EnglishToBliss": "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n", - "BlissToEnglish": "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" -} - # Create a formatted prompt template for an entry in the dataset # The "direction" parameter accepts either "EnglishToBliss" or "BlissToEnglish" diff --git a/jobs/Llama2/finetune/shared_data.py b/jobs/Llama2/finetune/shared_data.py new file mode 100644 index 0000000..eaafab8 --- /dev/null +++ b/jobs/Llama2/finetune/shared_data.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + +# Static instructions +instructions_map = { + "EnglishToBliss": "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n", + "BlissToEnglish": "### Instruction: \nConvert the input Bliss sentence to a English sentence.\n\n" +} diff --git a/jobs/stylegan3/job_stylegan3-t_gen_images.sh b/jobs/stylegan3/job_stylegan3-t_gen_images.sh index c9f68de..c265459 100755 --- a/jobs/stylegan3/job_stylegan3-t_gen_images.sh +++ b/jobs/stylegan3/job_stylegan3-t_gen_images.sh @@ -1,4 +1,13 @@ #!/bin/bash + +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + #SBATCH --job-name=stylegan3-t-gen-images #SBATCH --time 10-00:00 #SBATCH --nodes=1 diff --git a/jobs/stylegan3/job_stylegan3.sh b/jobs/stylegan3/job_stylegan3.sh index 56669b8..8f65b5b 100755 --- a/jobs/stylegan3/job_stylegan3.sh +++ b/jobs/stylegan3/job_stylegan3.sh @@ -1,4 +1,13 @@ #!/bin/bash + +# Copyright (c) 2023-2024, Inclusive Design Institute +# +# Licensed under the BSD 3-Clause License. You may not use this file except +# in compliance with this License. +# +# You may obtain a copy of the BSD 3-Clause License at +# https://github.com/inclusive-design/baby-bliss-bot/blob/main/LICENSE + #SBATCH --job-name=stylegan3 #SBATCH --time 24-00:00 #SBATCH --nodes=1 From a7f7179fcfea0ca9dfdab526569d524f09e6054e Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 22 Apr 2024 16:59:27 -0400 Subject: [PATCH 20/20] fix: remove an un-used var --- jobs/Llama2/finetune/eval_7b_hf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/Llama2/finetune/eval_7b_hf.py b/jobs/Llama2/finetune/eval_7b_hf.py index 8045c43..47b817d 100644 --- a/jobs/Llama2/finetune/eval_7b_hf.py +++ b/jobs/Llama2/finetune/eval_7b_hf.py @@ -46,7 +46,6 @@ def generate_text_with_prompt(prompt, model, tokenizer, temperature=0.7): # Test with exactly same instructions used in fine-tuning -instruction = "### Instruction: \nConvert the input English sentence to a Bliss sentence.\n\n" input = "I am a programmer." generate_text_with_instruction(instructions_map["EnglishToBliss"], input, model, tokenizer)