evaluation.yaml

# Config for running the InferenceRecipe in generate.py to generate output from an LLM
#
# This config assumes that you've run the following command before launching:
#   tune download meta-llama/Llama-3.2-11B-Vision-Instruct --output-dir /tmp/Llama-3.2-11B-Vision-Instruct
#
# It also assumes that you've downloaded the EleutherAI Eval Harness:
#   pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@fb963f0f0a5b28b69763590bb59676072cf43a01
#
# To launch, run the following command from root torchtune directory:
#    tune run eleuther_eval --config llama3_2_vision/evaluation

# Model arguments
model:
  _component_: torchtune.models.llama3_2_vision.llama3_2_vision_11b

# Transform arguments
tokenizer:
  _component_: torchtune.models.llama3_2_vision.llama3_2_vision_transform
  path: /tmp/Llama-3.2-11B-Vision-Instruct/original/tokenizer.model
  max_seq_len: 8192 # Limit the size of our inputs

# Checkpointer
checkpointer:
  _component_: torchtune.training.FullModelMetaCheckpointer
  checkpoint_dir: /tmp/Llama-3.2-11B-Vision-Instruct/original
  checkpoint_files: [consolidated.pth]
  output_dir: ./
  model_type: LLAMA3_VISION

# Environment
device: cuda
dtype: bf16
seed: 1234 # It is not recommended to change this seed, b/c it matches EleutherAI's default seed
log_level: INFO

# EleutherAI specific eval args
# Llama3.2 vision reports on MMMU Val using chain-of-thought reasoning
# and image concatenation. This is not currently supported in the EletherAI
# Eval Harness so results may not match the paper OOTB
tasks: ["mmmu_val_science"] # Defaulting to science as a good subset
limit: null
batch_size: 1
enable_kv_cache: True

# Quantization specific args
# Quantization is not supported in this specific config
quantizer: null