.github/workflows/update_pytorch.yml #2719
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # name: Run updater for pytorch that will check for pytorch packages | |
| on: | |
| schedule: | |
| # run the job every 30 minutes | |
| - cron: '*/30 * * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| jobs: | |
| generate_hash_letters: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| string_list: ${{ steps.generate.outputs.string_list }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' # Specify your Python version | |
| - uses: prefix-dev/setup-pixi@v0.5.1 | |
| with: | |
| pixi-version: "latest" | |
| manifest-path: pyproject.toml | |
| - name: pixi install | |
| run: | | |
| pixi install | |
| - name: Generate combination of missing subdirs and letters | |
| id: generate | |
| run: | | |
| # Run your script here to produce a list of missing subdirs and letters | |
| # example: linux-64@p linux-64@d | |
| message=$(pixi run parselmouth updater-producer --channel pytorch) | |
| if [ "$message" = "[]" ]; then | |
| echo "Skipping job as no new packages found" | |
| exit 0 | |
| fi | |
| echo $message | |
| echo "string_list=$message" >> $GITHUB_OUTPUT | |
| env: | |
| R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }} | |
| R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }} | |
| R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }} | |
| R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }} | |
| - name: Upload requested index.json | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: index | |
| path: output_index | |
| updater_of_records: | |
| runs-on: ubuntu-latest | |
| needs: generate_hash_letters | |
| strategy: | |
| matrix: | |
| subdirs: ${{fromJson(needs.generate_hash_letters.outputs.string_list)}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - uses: prefix-dev/setup-pixi@v0.5.1 | |
| with: | |
| pixi-version: "latest" | |
| manifest-path: pyproject.toml | |
| - name: pixi install | |
| run: | | |
| pixi install | |
| - name: Get partial index artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: index | |
| path: output_index | |
| - name: Get artifact info for subdir@letter and save into partial index | |
| run: pixi run parselmouth updater ${{ matrix.subdirs }} --channel pytorch --upload | |
| env: | |
| R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }} | |
| R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }} | |
| R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }} | |
| R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }} | |
| - name: Upload partial index | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: partial_index_${{ matrix.subdirs }} | |
| path: output | |
| updater_of_index: | |
| runs-on: ubuntu-latest | |
| needs: updater_of_records | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' # Specify your Python version | |
| - uses: prefix-dev/setup-pixi@v0.5.1 | |
| with: | |
| pixi-version: "latest" | |
| manifest-path: pyproject.toml | |
| - name: pixi install | |
| run: | | |
| pixi install | |
| - name: Get partial index artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| merge-multiple: true | |
| path: output | |
| - name: Combine all partial index and save into S3 index | |
| run: pixi run parselmouth updater-merger --upload --channel pytorch | |
| env: | |
| R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }} | |
| R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }} | |
| R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }} | |
| R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }} | |
| update_file: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # To push on main branch | |
| needs: updater_of_index | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - uses: prefix-dev/setup-pixi@v0.5.1 | |
| with: | |
| pixi-version: "latest" | |
| manifest-path: pyproject.toml | |
| - name: Update compressed mapping | |
| run: | | |
| # for pytorch we dont need to use legacy mapping anymore | |
| pixi run parselmouth update-mapping --channel pytorch | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }} | |
| R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }} | |
| R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }} | |
| R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }} | |
| - name: Commit and push changes | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git pull | |
| git add files/v0/pytorch | |
| git commit -m "Update file via GitHub Action" || true | |
| git push |