Import Data #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Import Data | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 2 1 */1 *' | |
| jobs: | |
| download-json: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout repo content | |
| uses: actions/checkout@v3 | |
| - name: pull changes from remote | |
| run: git pull origin | |
| - name: check repository size | |
| run: du -h -d 0 | |
| - name: download json file | |
| run: | | |
| wget --secure-protocol=TLSv1_2 -c --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 0 --continue -O "assets/weapons-wanted.json" "https://data.gov.ua/dataset/d0af9ba0-08b3-4bca-8508-02cffeaae8fd/resource/1fcab772-0b3c-4938-8f72-e60db343cbe5/download/weaponswanted.json" | |
| - name: upload json for pipeline job | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: weapons-wanted | |
| path: assets/weapons-wanted.json | |
| retention-days: 1 | |
| run-pipeline: | |
| runs-on: ubuntu-latest | |
| needs: download-json | |
| steps: | |
| - name: checkout repo content | |
| uses: actions/checkout@v3 | |
| - name: pull changes from remote | |
| run: git pull origin | |
| - name: display structure of the repository | |
| run: ls -R | |
| - name: setup python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: download json for pipeline job | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: weapons-wanted | |
| path: assets | |
| - name: execute py script | |
| run: python pipeline.py | |
| - name: check repository size | |
| run: du -h -d 0 | |
| - name: commit files | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add -A | |
| git diff-index --quiet HEAD || (git commit -a -m "Update main dataset" --allow-empty) | |
| - name: push changes | |
| uses: ad-m/github-push-action@v0.6.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: master |