Update workflows

.github/workflows/archive.yml

@@ -12,34 +12,35 @@ env:
   AUTOTHROTTLE_MAX_DELAY: 30.0
   AUTOTHROTTLE_START_DELAY: 1.5
   AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
+  PYTHON_VERSION: '3.9'
 
 jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
 
-      - name: Set up Python 3.8
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: ${{ env.PYTHON_VERSION }}  # Use the constant here
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
         uses: actions/cache@v1
         with:
           path: .venv
-          key: pip-3.8-${{ hashFiles('**/Pipfile.lock') }}
+          key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}  # Use the constant here
           restore-keys: |
-            pip-3.8-
+            pip-${{ env.PYTHON_VERSION }}-
             pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.8
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}  # Use the constant here
 
       - name: Run scrapers
         run: |

.github/workflows/ci.yml

@@ -28,14 +28,14 @@ jobs:
       - name: Install Pipenv
         uses: dschep/install-pipenv-action@v1
 
-      # - name: Cache Python dependencies
-      #   uses: actions/cache@v1
-      #   with:
-      #     path: .venv
-      #     key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}
-      #     restore-keys: |
-      #       pip-${{ matrix.python-version }}-
-      #       pip-
+      - name: Cache Python dependencies
+        uses: actions/cache@v1
+        with:
+          path: .venv
+          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}
+          restore-keys: |
+            pip-${{ matrix.python-version }}-
+            pip-
 
       - name: Install dependencies
         run: pipenv sync --dev
@@ -56,7 +56,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          pipenv run pytest
+          pipenv run pytest || [ $? -eq 5 ]
 
       - name: Validate output with scrapy
         if: github.event_name == 'pull_request'
@@ -66,3 +66,9 @@ jobs:
           git diff-index --name-only --diff-filter=d $(git merge-base HEAD ${{ github.base_ref }}) | \
           grep -Pio '(?<=/spiders/).*(?=\.py)' | \
           xargs pipenv run scrapy validate
+
+      - name: Prevent workflow deactivation
+        uses: gautamkrishnar/keepalive-workflow@v1
+        with:
+          committer_username: "citybureau-bot"
+          committer_email: "documenters@citybureau.org"

.github/workflows/cron.yml

@@ -2,58 +2,61 @@ name: Cron
 
 on:
   schedule:
-    # Set any time that you'd like scrapers to run (in UTC)
-    - cron: "29 6 * * *"
+    - cron: "27 6 * * *"
   workflow_dispatch:
 
 env:
   CI: true
   PIPENV_VENV_IN_PROJECT: true
   SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod
-  WAYBACK_ENABLED: true
   AUTOTHROTTLE_MAX_DELAY: 30.0
   AUTOTHROTTLE_START_DELAY: 1.5
   AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
-  # Add secrets for the platform you're using and uncomment here
-  # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  # S3_BUCKET: ${{ secrets.S3_BUCKET }}
   AZURE_ACCOUNT_KEY: ${{ secrets.AZURE_ACCOUNT_KEY }}
   AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }}
   AZURE_CONTAINER: ${{ secrets.AZURE_CONTAINER }}
   AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}
-  # GOOGLE_APPLICATION_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
-  # GCS_BUCKET = os.getenv("GCS_BUCKET")
-  # Setup Sentry, add the DSN to secrets and uncomment here
-  # SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+  OPENVPN_USER: ${{ secrets.OPENVPN_USER }}
+  OPENVPN_PASS: ${{ secrets.OPENVPN_PASS }}
+  OPENVPN_CONFIG: ${{ secrets.OPENVPN_CONFIG }}
+  PYTHON_VERSION: '3.9'
 
 jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v1
 
-      - name: Set up Python 3.9
+      - name: Connect to OpenVPN
+        run: |
+          sudo apt-get install -y openvpn
+          echo "$OPENVPN_USER" | sudo tee -a /etc/openvpn/client/auth
+          echo "$OPENVPN_PASS" | sudo tee -a /etc/openvpn/client/auth
+          echo "$OPENVPN_CONFIG" | sudo tee -a /etc/openvpn/ovpn.conf
+          sudo openvpn --config /etc/openvpn/ovpn.conf --daemon
+          sleep 120
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v1
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
         uses: actions/cache@v1
         with:
           path: .venv
-          key: pip-3.9-${{ hashFiles('**/Pipfile.lock') }}
-          restore-keys: |
-            pip-3.9-
+          key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
+            pip-${{ env.PYTHON_VERSION }}-
             pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.9
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} 
 
       - name: Run scrapers
         run: |
@@ -63,4 +66,5 @@ jobs:
       - name: Combine output feeds
         run: |
           export PYTHONPATH=$(pwd):$PYTHONPATH
-          pipenv run scrapy combinefeeds -s LOG_ENABLED=False
+          pipenv run scrapy combinefeeds -s LOG_ENABLED=True
+      AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}