Skip to content

Commit

Permalink
minor
Browse files Browse the repository at this point in the history
  • Loading branch information
@HarshTrivedi
HarshTrivedi committed Nov 27, 2024
1 parent cc1da99 commit 9ec606e
Showing 1 changed file with 120 additions and 25 deletions.
145 changes: 120 additions & 25 deletions .github/workflows/new-entry.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,110 @@ jobs:
with:
ref: ${{ steps.get-branch.outputs.branch }}

- name: Extract arguments from comment
id: extract_args
run: |
echo "Extracting arguments..."
comment="${{ github.event.comment.body }}"
python_version=$(echo "$comment" | grep -oP '(?<=--python )\S+')
appworld_version=$(echo "$comment" | grep -oP '(?<=--appworld )\S+')
experiment_prefix=$(echo "$comment" | grep -oP '(?<=--experiment-prefix )\S+')
replace_last_flag=$(echo "$comment" | grep -q -- '--replace-last' && echo "--replace-last" || echo "")
echo "Python version: $python_version"
echo "Appworld version: $appworld_version"
echo "Experiment names: ${{ env.experiment_prefix }}_test_normal and ${{ env.experiment_prefix }}_test_challenge"
echo "Replace last flag: $replace_last_flag"
echo "python_version=$python_version" >> $GITHUB_ENV
echo "appworld_version=$appworld_version" >> $GITHUB_ENV
echo "experiment_prefix=$experiment_prefix" >> $GITHUB_ENV
echo "replace_last_flag=$replace_last_flag" >> $GITHUB_ENV
- uses: astral-sh/setup-uv@v3
with:
version: "0.4.4"

- name: Set up Python
run: uv python install ${{ env.python_version }}

- name: Install venv
run: uv venv

- name: Install dependencies
run: |
uv pip install appworld==${{ env.appworld_version }}
uv run appworld install
- name: Download appworld data
run: uv run appworld download data

- name: Fetch main branch
run: git fetch origin main

- name: Verify PR file changes
run: |
echo "Checking PR for exactly two new files..."
experiment_prefix="${{ env.experiment_prefix }}"
expected_files=("experiments/outputs/${experiment_prefix}_test_challenge/leaderboard.bundle" "experiments/outputs/${experiment_prefix}_test_normal/leaderboard.bundle")
new_files=$(git diff --name-only origin/main..HEAD)
echo "Expected files:"
printf "%s\n" "${expected_files[@]}"
echo "New files in the PR:"
echo "$new_files"
# Sort and compare file lists
expected_sorted=$(printf "%s\n" "${expected_files[@]}")
actual_sorted=$(echo "$new_files" | sort)
if [[ "$expected_sorted" != "$actual_sorted" ]]; then
echo "Error: File list does not match the expected files."
echo "Expected:"
echo "$expected_sorted"
echo "Actual:"
echo "$actual_sorted"
exit 1
fi
echo "PR file check passed. The file list matches exactly."
- name: Download relevant files
run: |
# DIRECTORY=experiments/outputs/full_code_refl_deepseekcoder_test_normal
# mkdir ${DIRECTORY}
DIRECTORY=experiments/outputs/full_code_refl_deepseekcoder_test_normal
FILE_PATH=${DIRECTORY}/leaderboard.bundle
FILE_PATH=experiments/outputs/${experiment_prefix}_test_normal/leaderboard.bundle
curl -L -o ${FILE_PATH} https://github.com/stonybrooknlp/appworld-leaderboard/raw/${{ steps.get-branch.outputs.branch }}/${FILE_PATH}
du -sh ${FILE_PATH}
cat ${FILE_PATH}
FILE_PATH=experiments/outputs/${experiment_prefix}_test_challenge/leaderboard.bundle
curl -L -o ${FILE_PATH} https://github.com/stonybrooknlp/appworld-leaderboard/raw/${{ steps.get-branch.outputs.branch }}/${FILE_PATH}
- name: Unpack experiments
run: |
uv run appworld unpack ${{ env.experiment_prefix }}_test_normal
uv run appworld unpack ${{ env.experiment_prefix }}_test_challenge
- name: Run evaluations
run: |
uv run appworld evaluate ${{ env.experiment_prefix }}_test_normal test_normal
uv run appworld evaluate ${{ env.experiment_prefix }}_test_challenge test_challenge
- name: Make and add leaderboard entry
run: uv run appworld make ${{ env.experiment_prefix }}_test_normal ${{ env.experiment_prefix }}_test_challenge ${{env.replace_last_flag}}

- name: Comment with leaderboard entry
if: ${{ success() }}
uses: actions/github-script@v6
with:
script: |
const fs = require('fs');
const entries = JSON.parse(fs.readFileSync('leaderboard.json', 'utf8'));
const formattedEntry = '```json\n' + JSON.stringify(entries[entries.length - 1], null, 4) + '\n```';
const commentBody = `### Latest Leaderboard Entry\n${formattedEntry}`;
const issue_number = context.issue.number;
await github.rest.issues.createComment({
...context.repo,
issue_number: issue_number,
body: commentBody,
});
# # - name: Checkout repository
# # uses: actions/checkout@v4
Expand Down Expand Up @@ -264,25 +359,25 @@ jobs:



# # # - name: Extract arguments from comment
# # # id: extract_args
# # # run: |
# # # echo "Extracting arguments..."
# # # comment="${{ github.event.comment.body }}"
# # # python_version=$(echo "$comment" | grep -oP '(?<=--python )\S+')
# # # appworld_version=$(echo "$comment" | grep -oP '(?<=--appworld )\S+')
# # # experiment_prefix=$(echo "$comment" | grep -oP '(?<=--experiment-prefix )\S+')
# # # replace_last_flag=$(echo "$comment" | grep -q -- '--replace-last' && echo "--replace-last" || echo "")

# # # echo "Python version: $python_version"
# # # echo "Appworld version: $appworld_version"
# # # echo "Experiment names: ${{ env.experiment_prefix }}_test_normal and ${{ env.experiment_prefix }}_test_challenge"
# # # echo "Replace last flag: $replace_last_flag"

# # # echo "python_version=$python_version" >> $GITHUB_ENV
# # # echo "appworld_version=$appworld_version" >> $GITHUB_ENV
# # # echo "experiment_prefix=$experiment_prefix" >> $GITHUB_ENV
# # # echo "replace_last_flag=$replace_last_flag" >> $GITHUB_ENV
- name: Extract arguments from comment
id: extract_args
run: |
echo "Extracting arguments..."
comment="${{ github.event.comment.body }}"
python_version=$(echo "$comment" | grep -oP '(?<=--python )\S+')
appworld_version=$(echo "$comment" | grep -oP '(?<=--appworld )\S+')
experiment_prefix=$(echo "$comment" | grep -oP '(?<=--experiment-prefix )\S+')
replace_last_flag=$(echo "$comment" | grep -q -- '--replace-last' && echo "--replace-last" || echo "")
echo "Python version: $python_version"
echo "Appworld version: $appworld_version"
echo "Experiment names: ${{ env.experiment_prefix }}_test_normal and ${{ env.experiment_prefix }}_test_challenge"
echo "Replace last flag: $replace_last_flag"
echo "python_version=$python_version" >> $GITHUB_ENV
echo "appworld_version=$appworld_version" >> $GITHUB_ENV
echo "experiment_prefix=$experiment_prefix" >> $GITHUB_ENV
echo "replace_last_flag=$replace_last_flag" >> $GITHUB_ENV
# # # - name: Checkout PR branch
Expand Down

0 comments on commit 9ec606e

Please sign in to comment.