Scheduled Run #227
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Process Flu SRA | |
| run-name: ${{ github.event.inputs.reason || 'Scheduled Run' }} | |
| on: | |
| schedule: | |
| - cron: '0 17 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| reason: | |
| description: 'Reason for running the workflow' | |
| required: true | |
| default: 'Routine Processing' | |
| concurrency: | |
| group: ${{ github.repository }} | |
| env: | |
| NXF_VER: "24.10.3" | |
| NXF_WORK: ${{ github.workspace }}/work | |
| NXF_OUTPUT: ${{ github.workspace }}/outputs | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| FLUSRA_VERSION: 'main' | |
| FILES_PER_CHUNK: 5 | |
| BIOPROJECTS: "PRJNA1102327,PRJNA1122849" | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| jobs: | |
| fetch_sra: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| chunk_files: ${{ steps.split_sra.outputs.chunk_files }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/actions/install-dependencies | |
| - name: Run nextflow | |
| run: nextflow run https://github.com/gp201/flusra.git -r ${{ env.FLUSRA_VERSION }} -c ${{ github.workspace }}/config/nextflow.config -profile mamba --only_fetch true --bioproject ${{ env.BIOPROJECTS }} --outdir ${{ env.NXF_OUTPUT }} | |
| - name: Split SRAs into chunks | |
| if: ${{ hashFiles('outputs/metadata/SraRunTable_automated_new.txt') != '' }} | |
| id: split_sra | |
| run: | | |
| mkdir -p ${{ github.workspace }}/sra_chunks/ | |
| grep -vxFf ${{ env.NXF_OUTPUT }}/metadata/SraRunTable_automated_milk_samples.txt ${{ env.NXF_OUTPUT }}/metadata/SraRunTable_automated_new.txt > ${{ env.NXF_OUTPUT }}/metadata/SraRunTable_without_milk_samples.txt | |
| split -l ${{ env.FILES_PER_CHUNK }} ${{ env.NXF_OUTPUT }}/metadata/SraRunTable_without_milk_samples.txt ${{ github.workspace }}/sra_chunks/chunk_ | |
| split -l ${{ env.FILES_PER_CHUNK }} ${{ env.NXF_OUTPUT }}/metadata/SraRunTable_automated_milk_samples.txt ${{ github.workspace }}/sra_chunks/milk_chunk_ | |
| chunk_files=$(ls ${{ github.workspace }}/sra_chunks/ | jq -R . | jq -s . | tr -d '\n' | tr -d " ") | |
| echo "chunk_files=$chunk_files" >> $GITHUB_OUTPUT | |
| - name: Upload chunk files | |
| if: ${{ steps.split_sra.outputs.chunk_files != '' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sra_chunks | |
| path: ${{ github.workspace }}/sra_chunks/ | |
| - name: Upload Metadata | |
| if: ${{ steps.split_sra.outputs.chunk_files != '' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: metadata | |
| path: ${{ env.NXF_OUTPUT }}/metadata/SraRunTable_automated_updated.csv | |
| - name: Upload Nextflow log | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: failed-run-fetch-sra | |
| include-hidden-files: true | |
| path: | | |
| ${{ github.workspace }}/.nextflow.log | |
| process_sra: | |
| needs: fetch_sra | |
| if: ${{ needs.fetch_sra.outputs.chunk_files != '' }} | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| chunk_file: ${{fromJson(needs.fetch_sra.outputs.chunk_files)}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Dependencies | |
| uses: ./.github/actions/install-dependencies | |
| - name: Download chunk files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sra_chunks | |
| path: ${{ github.workspace }}/sra_chunks/ | |
| - name: Setup samples | |
| run: | | |
| # if milk in chunk_file name, then set github env as IS_MILK_SAMPLE = true | |
| if [[ ${{ matrix.chunk_file }} == *"milk"* ]]; then | |
| echo "IS_MILK_SAMPLE=true" >> $GITHUB_ENV | |
| else | |
| echo "IS_MILK_SAMPLE=false" >> $GITHUB_ENV | |
| fi | |
| - name: Run nextflow | |
| if: ${{ env.IS_MILK_SAMPLE != 'true' }} | |
| run: nextflow run https://github.com/gp201/flusra.git -r ${{ env.FLUSRA_VERSION }} -c ${{ github.workspace }}/config/nextflow.config -profile mamba --sra_accessions ${{ github.workspace }}/sra_chunks/${{ matrix.chunk_file }} --outdir ${{ env.NXF_OUTPUT }} | |
| - name: Run nextflow for milk samples | |
| if: ${{ env.IS_MILK_SAMPLE == 'true' }} | |
| run: nextflow run https://github.com/gp201/flusra.git -r ${{ env.FLUSRA_VERSION }} -c ${{ github.workspace }}/config/nextflow.config -profile mamba --sra_accessions ${{ github.workspace }}/sra_chunks/${{ matrix.chunk_file }} --milk_sra_accessions ${{ github.workspace }}/sra_chunks/${{ matrix.chunk_file }} --outdir ${{ env.NXF_OUTPUT }} | |
| - name: Upload Outputs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: files-${{ matrix.chunk_file }} | |
| path: | | |
| ${{ env.NXF_OUTPUT }}/fasta/ | |
| ${{ env.NXF_OUTPUT }}/depth/ | |
| ${{ env.NXF_OUTPUT }}/variants/ | |
| - name: Upload Demixed | |
| if: ${{ env.IS_MILK_SAMPLE == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: demixed-${{ matrix.chunk_file }} | |
| path: | | |
| ${{ env.NXF_OUTPUT }}/demixed/ | |
| - name: Upload Nextflow log | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: failed-run-${{ matrix.chunk_file }} | |
| include-hidden-files: true | |
| path: | | |
| ${{ github.workspace }}/.nextflow.log | |
| push_files: | |
| needs: [process_sra] | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download metadata | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: metadata | |
| path: ${{ env.NXF_OUTPUT }}/ | |
| - name: Download files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: files-* | |
| path: ${{ env.NXF_OUTPUT }}/ | |
| merge-multiple: true | |
| - name: Download demixed files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: demixed-* | |
| path: ${{ env.NXF_OUTPUT }}/demixed/ | |
| merge-multiple: true | |
| - name: Push outputs | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| mv ${{ env.NXF_OUTPUT }}/SraRunTable_automated_updated.csv ${{ github.workspace }}/metadata/SraRunTable_automated.csv | |
| mv ${{ env.NXF_OUTPUT }}/fasta/*.fa ${{ github.workspace }}/fasta/ | |
| mv ${{ env.NXF_OUTPUT }}/depth/*.tsv ${{ github.workspace }}/depth/ | |
| mv ${{ env.NXF_OUTPUT }}/variants/*.tsv ${{ github.workspace }}/variants/ | |
| if [ -d ${{ env.NXF_OUTPUT }}/demixed/ ]; then | |
| mv ${{ env.NXF_OUTPUT }}/demixed/* ${{ github.workspace }}/demixed/ | |
| git add ${{ github.workspace }}/demixed/ | |
| fi | |
| git add ${{ github.workspace }}/metadata/SraRunTable_automated.csv | |
| git add ${{ github.workspace }}/fasta/*.fa | |
| git add ${{ github.workspace }}/depth/*.tsv | |
| git add ${{ github.workspace }}/variants/*.tsv | |
| git commit -m "Add consensus sequences, depth, variant files, demixed files and updated metadata" | |
| git push |