Update workflows

City-Bureau · Apr 25, 2024 · 1f75f0b · 1f75f0b
1 parent 1fd8b99
commit 1f75f0b
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 40 deletions.
diff --git a/.github/workflows/archive.yml b/.github/workflows/archive.yml
@@ -7,40 +7,40 @@ on:
 
 env:
   CI: true
+  PYTHON_VERSION: 3.9
   PIPENV_VENV_IN_PROJECT: true
   SCRAPY_SETTINGS_MODULE: city_scrapers.settings.archive
   AUTOTHROTTLE_MAX_DELAY: 30.0
   AUTOTHROTTLE_START_DELAY: 1.5
   AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
-  PYTHON_VERSION: '3.9'
 
 jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}  # Use the constant here
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Pipenv
         run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
-          key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}  # Use the constant here
+          key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
           restore-keys: |
-            pip-${{ env.PYTHON_VERSION }}-
+            ${{ env.PYTHON_VERSION }}-
             pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}  # Use the constant here
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
 
       - name: Run scrapers
         run: |

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -18,18 +18,18 @@ jobs:
         python-version: [3.9]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
           key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}
@@ -55,6 +55,7 @@ jobs:
           pipenv run flake8 .
 
       - name: Test with pytest
+        # Ignores exit code 5 (no tests collected)
         run: |
           pipenv run pytest || [ $? -eq 5 ]
 
@@ -66,9 +67,3 @@ jobs:
           git diff-index --name-only --diff-filter=d $(git merge-base HEAD ${{ github.base_ref }}) | \
           grep -Pio '(?<=/spiders/).*(?=\.py)' | \
           xargs pipenv run scrapy validate
-
-      - name: Prevent workflow deactivation
-        uses: gautamkrishnar/keepalive-workflow@v1
-        with:
-          committer_username: "citybureau-bot"
-          committer_email: "documenters@citybureau.org"
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -2,13 +2,15 @@ name: Cron
 
 on:
   schedule:
-    - cron: "27 6 * * *"
+    - cron: "1 6 * * *"
   workflow_dispatch:
 
 env:
   CI: true
+  PYTHON_VERSION: 3.9
   PIPENV_VENV_IN_PROJECT: true
   SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod
+  WAYBACK_ENABLED: true
   AUTOTHROTTLE_MAX_DELAY: 30.0
   AUTOTHROTTLE_START_DELAY: 1.5
   AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
@@ -17,46 +19,34 @@ env:
   AZURE_CONTAINER: ${{ secrets.AZURE_CONTAINER }}
   AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
-  OPENVPN_USER: ${{ secrets.OPENVPN_USER }}
-  OPENVPN_PASS: ${{ secrets.OPENVPN_PASS }}
-  OPENVPN_CONFIG: ${{ secrets.OPENVPN_CONFIG }}
-  PYTHON_VERSION: '3.9'
 
 jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-
-      - name: Connect to OpenVPN
-        run: |
-          sudo apt-get install -y openvpn
-          echo "$OPENVPN_USER" | sudo tee -a /etc/openvpn/client/auth
-          echo "$OPENVPN_PASS" | sudo tee -a /etc/openvpn/client/auth
-          echo "$OPENVPN_CONFIG" | sudo tee -a /etc/openvpn/ovpn.conf
-          sudo openvpn --config /etc/openvpn/ovpn.conf --daemon
-          sleep 120
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Pipenv
         run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
-          key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
-            pip-${{ env.PYTHON_VERSION }}-
+          key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
+          restore-keys: |
+            ${{ env.PYTHON_VERSION }}-
             pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} 
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
 
       - name: Run scrapers
         run: |
@@ -66,5 +56,10 @@ jobs:
       - name: Combine output feeds
         run: |
           export PYTHONPATH=$(pwd):$PYTHONPATH
-          pipenv run scrapy combinefeeds -s LOG_ENABLED=True
-      AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}
+          pipenv run scrapy combinefeeds -s LOG_ENABLED=False
+
+      - name: Prevent workflow deactivation
+        uses: gautamkrishnar/keepalive-workflow@v1
+        with:
+          committer_username: "citybureau-bot"
+          committer_email: "documenters@citybureau.org"