From 1f75f0b8cf01e450a5b9f663c60ccb89151f87d3 Mon Sep 17 00:00:00 2001 From: Daniel Simmons-Ritchie <37225902+SimmonsRitchie@users.noreply.github.com> Date: Thu, 25 Apr 2024 10:48:02 -0500 Subject: [PATCH] Update workflows --- .github/workflows/archive.yml | 16 +++++++------- .github/workflows/ci.yml | 15 +++++--------- .github/workflows/cron.yml | 39 +++++++++++++++-------------------- 3 files changed, 30 insertions(+), 40 deletions(-) diff --git a/.github/workflows/archive.yml b/.github/workflows/archive.yml index 022b40a..a6c1464 100644 --- a/.github/workflows/archive.yml +++ b/.github/workflows/archive.yml @@ -7,40 +7,40 @@ on: env: CI: true + PYTHON_VERSION: 3.9 PIPENV_VENV_IN_PROJECT: true SCRAPY_SETTINGS_MODULE: city_scrapers.settings.archive AUTOTHROTTLE_MAX_DELAY: 30.0 AUTOTHROTTLE_START_DELAY: 1.5 AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0 - PYTHON_VERSION: '3.9' jobs: crawl: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_VERSION }} # Use the constant here + python-version: ${{ env.PYTHON_VERSION }} - name: Install Pipenv run: pip install --user pipenv - name: Cache Python dependencies - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: .venv - key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} # Use the constant here + key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} restore-keys: | - pip-${{ env.PYTHON_VERSION }}- + ${{ env.PYTHON_VERSION }}- pip- - name: Install dependencies run: pipenv sync env: - PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} # Use the constant here + PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - name: Run scrapers run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c430ed5..ccb9a16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,18 +18,18 @@ jobs: python-version: [3.9] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Pipenv - uses: dschep/install-pipenv-action@v1 + run: pip install --user pipenv - name: Cache Python dependencies - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: .venv key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }} @@ -55,6 +55,7 @@ jobs: pipenv run flake8 . - name: Test with pytest + # Ignores exit code 5 (no tests collected) run: | pipenv run pytest || [ $? -eq 5 ] @@ -66,9 +67,3 @@ jobs: git diff-index --name-only --diff-filter=d $(git merge-base HEAD ${{ github.base_ref }}) | \ grep -Pio '(?<=/spiders/).*(?=\.py)' | \ xargs pipenv run scrapy validate - - - name: Prevent workflow deactivation - uses: gautamkrishnar/keepalive-workflow@v1 - with: - committer_username: "citybureau-bot" - committer_email: "documenters@citybureau.org" diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 2ccb37e..b761f83 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -2,13 +2,15 @@ name: Cron on: schedule: - - cron: "27 6 * * *" + - cron: "1 6 * * *" workflow_dispatch: env: CI: true + PYTHON_VERSION: 3.9 PIPENV_VENV_IN_PROJECT: true SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod + WAYBACK_ENABLED: true AUTOTHROTTLE_MAX_DELAY: 30.0 AUTOTHROTTLE_START_DELAY: 1.5 AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0 @@ -17,28 +19,15 @@ env: AZURE_CONTAINER: ${{ secrets.AZURE_CONTAINER }} AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - OPENVPN_USER: ${{ secrets.OPENVPN_USER }} - OPENVPN_PASS: ${{ secrets.OPENVPN_PASS }} - OPENVPN_CONFIG: ${{ secrets.OPENVPN_CONFIG }} - PYTHON_VERSION: '3.9' jobs: crawl: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - - name: Connect to OpenVPN - run: | - sudo apt-get install -y openvpn - echo "$OPENVPN_USER" | sudo tee -a /etc/openvpn/client/auth - echo "$OPENVPN_PASS" | sudo tee -a /etc/openvpn/client/auth - echo "$OPENVPN_CONFIG" | sudo tee -a /etc/openvpn/ovpn.conf - sudo openvpn --config /etc/openvpn/ovpn.conf --daemon - sleep 120 + - uses: actions/checkout@v4 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -46,17 +35,18 @@ jobs: run: pip install --user pipenv - name: Cache Python dependencies - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: .venv - key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} - pip-${{ env.PYTHON_VERSION }}- + key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} + restore-keys: | + ${{ env.PYTHON_VERSION }}- pip- - name: Install dependencies run: pipenv sync env: - PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - name: Run scrapers run: | @@ -66,5 +56,10 @@ jobs: - name: Combine output feeds run: | export PYTHONPATH=$(pwd):$PYTHONPATH - pipenv run scrapy combinefeeds -s LOG_ENABLED=True - AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }} + pipenv run scrapy combinefeeds -s LOG_ENABLED=False + + - name: Prevent workflow deactivation + uses: gautamkrishnar/keepalive-workflow@v1 + with: + committer_username: "citybureau-bot" + committer_email: "documenters@citybureau.org"