Identifying Variables Eval #2238

	name: Run new evals

	on:
	workflow_dispatch:
	pull_request:
	branches:
	- main
	paths:
	- 'evals/registry/**'

	jobs:
	check_files:
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v2
	with:
	fetch-depth: 0
	lfs: true

	- name: Install Git LFS
	run: \|
	sudo apt-get install git-lfs
	git lfs install

	- name: Set up Python
	uses: actions/setup-python@v2
	with:
	python-version: 3.9

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install pyyaml
	pip install -e .

	- name: Get list of new YAML files in evals/registry/evals
	id: get_files
	run: \|
	# Use environment files to store the output
	git diff --name-only --diff-filter=A ${{ github.event.pull_request.base.sha }} ${{ github.sha }} \| grep '^evals/registry/evals/.*\.yaml$' \| xargs > new_files
	echo "new_files=$(cat new_files)" >> $GITHUB_ENV

	- name: Run oaieval command for each new YAML file
	env:
	OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
	run: \|
	files="${{ env.new_files }}"
	if [ -n "$files" ]; then
	for file in $files; do
	echo "Processing $file"
	first_key=$(python .github/workflows/parse_yaml.py $file)
	echo "Eval Name: $first_key"
	oaieval dummy $first_key --max_samples 10
	done
	else
	echo "No new YAML files found in evals/registry/evals"
	fi

Provide feedback