GH Task Runner (MPS) #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GH Task Runner (MPS) | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| run_task: | ||
| description: 'Task to run' | ||
| required: true | ||
| default: 'arc' | ||
| options: | ||
| - anli | ||
| - arc | ||
| - arithmetic | ||
| - asdiv | ||
| - babi | ||
| - bbh | ||
| - belebele | ||
| - benchmarks | ||
| - bigbench | ||
| - blimp | ||
| - ceval | ||
| - cmmlu | ||
| - code_x_glue | ||
| - coqa | ||
| - crows_pairs | ||
| - csatqa | ||
| - drop | ||
| - fld | ||
| - glue | ||
| - gsm8k | ||
| - headqa | ||
| - hellaswag | ||
| - hendrycks_ethics | ||
| - ifeval | ||
| - kmmlu | ||
| - kobest | ||
| - lambada | ||
| - lambada_cloze | ||
| - lambada_multilingual | ||
| - logiqa | ||
| - logiqa2 | ||
| - mathqa | ||
| - mc_taco | ||
| - medmcqa | ||
| - medqa | ||
| - mgsm | ||
| - minerva_math | ||
| - model_written_evals | ||
| - mutual | ||
| - nq_open | ||
| - okapi | ||
| - openbookqa | ||
| - paws-x | ||
| - piqa | ||
| - pile | ||
| - polemo2 | ||
| - prost | ||
| - pubmedqa | ||
| - qa4mre | ||
| - qasper | ||
| - race | ||
| - realtoxicityprompts | ||
| - sciq | ||
| - scrolls | ||
| - siqa | ||
| - squadv2 | ||
| - storycloze | ||
| - super_glue | ||
| - swag | ||
| - toxigen | ||
| - translation | ||
| - triviaqa | ||
| - truthfulqa | ||
| - unscramble | ||
| - webqs | ||
| - wikitext | ||
| - winogrande | ||
| - wmt2016 | ||
| - wsc273 | ||
| - xcopa | ||
| - xnli | ||
| - xstorycloze | ||
| - xwinograd | ||
| custom_task: | ||
| description: 'Custom Task to run (overwrites previous)' | ||
| required: false | ||
| default: '' | ||
| num_fewshot: | ||
| description: 'num_fewshot setting (ignored if < 0)' | ||
| required: true | ||
| default: -1 | ||
| model_hf_repo: | ||
| description: 'Model Hugging Face Repository' | ||
| required: true | ||
| default: 'RWKV/rwkv-5-world-1b5' | ||
| model_args: | ||
| description: 'Model Arguments' | ||
| required: false | ||
| default: 'dtype="float32",trust_remote_code=True' | ||
| batch_size: | ||
| description: 'Batch Size' | ||
| required: true | ||
| default: 'auto' | ||
| # backend: | ||
| # description: 'Backend to use' | ||
| # required: true | ||
| # default: 'mps' | ||
| # options: | ||
| # - nvidia-gpu | ||
| # - intel-gpu | ||
| # - amd-gpu | ||
| # - any-gpu | ||
| # gpu_vram: | ||
| # description: 'Minimum GPU VRAM (ignored for MPS)' | ||
| # required: true | ||
| # default: '24' | ||
| # options: | ||
| # - 16 | ||
| # - 24 | ||
| # - 80 | ||
| env: | ||
| # Get the final task | ||
| RUN_TASK: ${{ github.event.inputs.custom_task || github.event.inputs.run_task }} | ||
| # HF repo to sync to | ||
| HF_REPO_SYNC: rwkv-x-dev/lm-eval-output | ||
| # Model HF repo | ||
| MODEL_HF_REPO: ${{ github.event.inputs.model_hf_repo }} | ||
| # Secrets | ||
| HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}} | ||
| jobs: | ||
| gh-task-runner-mps: | ||
| # Name of the job | ||
| name: "[MPS-${{ github.event.inputs.model_hf_repo }}] ${{ github.event.inputs.custom_task || github.event.inputs.run_task }} - ${{ github.event.inputs.model_args }}" | ||
| # Due to github worker hard limitation, of 24 hours | ||
| # we apply a timeout of 23 hours instead. | ||
| timeout-minutes: 1380 | ||
| # Currently, MacOS ARM is only on v14. | ||
| # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners | ||
| runs-on: macos-14 | ||
| # Actual task setup, and run steps | ||
| steps: | ||
| - name: Clone repository | ||
| id: checkout | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| submodules: 'recursive' | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: '3.11' | ||
| - name: Install dependencies / setup project | ||
| run: | | ||
| pip install -e . | ||
| mkdir -p ./output | ||
| - name: Run Task | ||
| run: | | ||
| # Get the final task to run | ||
| task_to_run=${{ github.event.inputs.custom_task }} | ||
| if [ -z "$task_to_run" ]; then | ||
| task_to_run=${{ github.event.inputs.run_task }} | ||
| fi | ||
| # Check if the few shot setting is larger or euqal to 0 | ||
| if [ ${{ github.event.inputs.num_fewshot }} -ge 0 ]; then | ||
| lm_eval --model hf \ | ||
| --model_args pretrained=${{ github.event.inputs.model_hf_repo }},${{ github.event.inputs.model_args }} \ | ||
| --tasks $task_to_run \ | ||
| --batch_size ${{ github.event.inputs.batch_size }} \ | ||
| --device mps \ | ||
| --num_fewshot ${{ github.event.inputs.num_fewshot }} \ | ||
| --log_samples --output_path ./output 2>&1 | tee -a ./output/run-log.txt | ||
| else | ||
| lm_eval --model hf \ | ||
| --model_args pretrained=${{ github.event.inputs.model_hf_repo }},${{ github.event.inputs.model_args }} \ | ||
| --tasks $task_to_run \ | ||
| --batch_size ${{ github.event.inputs.batch_size }} \ | ||
| --device mps \ | ||
| --log_samples --output_path ./output 2>&1 | tee -a ./output/run-log.txt | ||
| fi | ||
| - name: Upload outputs to HF | ||
| if: always() | ||
| run: | | ||
|
Check failure on line 192 in .github/workflows/gh-task-runner-mps.yml
|
||
| ./gh-task-runner/hf-upload-runner.sh "${{ env.HF_REPO_SYNC }}" "${{ env.MODEL_HF_REPO }}/${{ env.RUN_TASK//'*'/'_' }}/MPS/${{ github.event.inputs.model_args }}" "./output" | ||
| # Note that this is meant to be a contigency measure, in case the HF upload failed | ||
| - name: Save output models | ||
| uses: actions/upload-artifact@v3 | ||
| # if: failure() | ||
| if: always() | ||
| with: | ||
| name: output-files | ||
| path: | | ||
| output/* | ||
| # retention-days: 90 | ||
