get-and-transform-data.yml

name: Get and transform latest covid therapeutics data

on:
  workflow_dispatch:
  schedule:
    - cron:  '6 * * * *'

jobs:
  scheduled:
    runs-on: ubuntu-latest
    steps:
    - name: Check out this repo
      uses: actions/checkout@v2
      with:
        depth: 1 

    - name: Install Python Dependencies (PIP)
      run: |-
        pip install pandas
      shell: sh
    - name: Fetch latest dose data
      run: |-
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/rxn6-qnx8.csv?\$limit=50000&order_label=Evusheld" > data/therapeutics/evusheld/evusheld-providers.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/rxn6-qnx8.csv?\$limit=50000&order_label=Paxlovid" > data/therapeutics/paxlovid/paxlovid-providers.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/rxn6-qnx8.csv?\$limit=50000&order_label=Renal+Paxlovid" > data/therapeutics/renal-paxlovid/renal-paxlovid-providers.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/rxn6-qnx8.csv?\$limit=50000&order_label=Bebtelovimab" > data/therapeutics/bebtelovimab/bebtelovimab-providers.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/rxn6-qnx8.csv?\$limit=50000&order_label=Lagevrio%20(molnupiravir)" > data/therapeutics/lagevrio/lagevrio-providers.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/rxn6-qnx8.csv?\$limit=50000&order_label=Sotrovimab" > data/therapeutics/sotrovimab/sotrovimab-providers.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/rxn6-qnx8.csv?\$limit=50000&order_label=Commercial%20Veklury%20(remdesivir)" > data/therapeutics/remdesivir/remdesivir-providers-aspr.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/6m8a-tsjg.csv?\$limit=60000" > data/therapeutics/testToTreat/testToTreat-providers.csv
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/j7fh-jg79.json" > data/therapeutics/publish-events.json
        curl --header "X-App-Token: ${{ secrets.HEALTHDATA_APPTOKEN }}" "https://healthdata.gov/resource/savg-57kf.json" > data/therapeutics/testToTreat/publish-events.json
    
    - name: Sort therapeutics data by state, county, city, provider
      run: |-
        python src/transform-mAbs-data.py data/therapeutics/evusheld/evusheld-providers.csv
        python src/transform-mAbs-data.py data/therapeutics/paxlovid/paxlovid-providers.csv
        python src/transform-mAbs-data.py data/therapeutics/remdesivir/remdesivir-providers-aspr.csv
        python src/transform-mAbs-data.py data/therapeutics/remdesivir/remdesivir-providers-aspr.csv data/therapeutics/remdesivir/remdesivir-providers-covidsafe-fyi.csv data/therapeutics/remdesivir/remdesivir-providers.csv
        python src/transform-mAbs-data.py data/therapeutics/renal-paxlovid/renal-paxlovid-providers.csv
        python src/transform-mAbs-data.py data/therapeutics/bebtelovimab/bebtelovimab-providers.csv
        python src/transform-mAbs-data.py data/therapeutics/lagevrio/lagevrio-providers.csv
        python src/transform-mAbs-data.py data/therapeutics/testToTreat/testToTreat-providers.csv
      shell: sh
    - name: Commit and push if it changed
      run: |-
        git config user.name "Automated"
        git config user.email "actions@users.noreply.github.com"
        git add -A
        timestamp=$(date -u)
        git commit -m "Latest therapeutics data: ${timestamp}" || exit 0
        git push

    - name: Capture latest covid.gov
      run: |-
        curl "https://www.covid.gov" > data/covid-info/covid.gov/index.html
        curl "https://www.covid.gov/longcovid" > data/covid-info/covid.gov/longcovid/index.html
    - name: Commit and push if it changed
      run: |-
        git config user.name "Automated"
        git config user.email "actions@users.noreply.github.com"
        git add -A
        timestamp=$(date -u)
        git commit -m "Latest covid.gov: ${timestamp}" || exit 0
        git push
        
    - name: Update zip code inventory history for all mAbs
      run: |-
        python src/extract-dose-history.py
    - name: Commit and push if it changed
      run: |-
        git config user.name "Automated"
        git config user.email "actions@users.noreply.github.com"
        git add -A
        timestamp=$(date -u)
        git commit -m "Latest inventory by zip code: ${timestamp}" || exit 0
        git push
    - name: Update dose history totals for all mAbs
      run: |-        
        python src/dose-history-totals.py
      shell: sh
    - name: Commit and push if it changed
      run: |-
        git config user.name "Automated"
        git config user.email "actions@users.noreply.github.com"
        git add -A
        timestamp=$(date -u)
        git commit -m "Latest dose history totals: ${timestamp}" || exit 0
        git push


    - name: download state population info from census.gov
      run: |-
        curl https://www2.census.gov/programs-surveys/popest/datasets/2020-2021/state/totals/NST-EST2021-alldata.csv > data/states/state-populations.csv
    - name: download county adjacency info from census.gov
      run: |-
        curl https://www2.census.gov/geo/docs/reference/county_adjacency.txt > data/counties/county_adjacency.txt
    - name: Commit and push if it changed
      run: |-
        git config user.name "Automated"
        git config user.email "actions@users.noreply.github.com"
        git add -A
        timestamp=$(date -u)
        git commit -m "Latest state and county data: ${timestamp}" || exit 0
        git push