diff --git a/.github/workflows/archive.yml b/.github/workflows/archive.yml index a512110..a6c1464 100644 --- a/.github/workflows/archive.yml +++ b/.github/workflows/archive.yml @@ -7,6 +7,7 @@ on: env: CI: true + PYTHON_VERSION: 3.9 PIPENV_VENV_IN_PROJECT: true SCRAPY_SETTINGS_MODULE: city_scrapers.settings.archive AUTOTHROTTLE_MAX_DELAY: 30.0 @@ -17,29 +18,29 @@ jobs: crawl: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: ${{ env.PYTHON_VERSION }} - name: Install Pipenv - uses: dschep/install-pipenv-action@v1 + run: pip install --user pipenv - name: Cache Python dependencies - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: .venv - key: pip-3.8-${{ hashFiles('**/Pipfile.lock') }} + key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} restore-keys: | - pip-3.8- + ${{ env.PYTHON_VERSION }}- pip- - name: Install dependencies run: pipenv sync env: - PIPENV_DEFAULT_PYTHON_VERSION: 3.8 + PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - name: Run scrapers run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba05240..ccb9a16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,24 +18,24 @@ jobs: python-version: [3.9] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Pipenv - uses: dschep/install-pipenv-action@v1 + run: pip install --user pipenv - # - name: Cache Python dependencies - # uses: actions/cache@v1 - # with: - # path: .venv - # key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }} - # restore-keys: | - # pip-${{ matrix.python-version }}- - # pip- + - name: Cache Python dependencies + uses: actions/cache@v4 + with: + path: .venv + key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }} + restore-keys: | + pip-${{ matrix.python-version }}- + pip- - name: Install dependencies run: pipenv sync --dev @@ -55,8 +55,9 @@ jobs: pipenv run flake8 . - name: Test with pytest + # Ignores exit code 5 (no tests collected) run: | - pipenv run pytest + pipenv run pytest || [ $? -eq 5 ] - name: Validate output with scrapy if: github.event_name == 'pull_request' diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 91bcbe8..b761f83 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -2,58 +2,51 @@ name: Cron on: schedule: - # Set any time that you'd like scrapers to run (in UTC) - - cron: "29 6 * * *" + - cron: "1 6 * * *" workflow_dispatch: env: CI: true + PYTHON_VERSION: 3.9 PIPENV_VENV_IN_PROJECT: true SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod WAYBACK_ENABLED: true AUTOTHROTTLE_MAX_DELAY: 30.0 AUTOTHROTTLE_START_DELAY: 1.5 AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0 - # Add secrets for the platform you're using and uncomment here - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # S3_BUCKET: ${{ secrets.S3_BUCKET }} AZURE_ACCOUNT_KEY: ${{ secrets.AZURE_ACCOUNT_KEY }} AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }} AZURE_CONTAINER: ${{ secrets.AZURE_CONTAINER }} AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }} - # GOOGLE_APPLICATION_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - # GCS_BUCKET = os.getenv("GCS_BUCKET") - # Setup Sentry, add the DSN to secrets and uncomment here - # SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + SENTRY_DSN: ${{ secrets.SENTRY_DSN }} jobs: crawl: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v1 + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - name: Install Pipenv - uses: dschep/install-pipenv-action@v1 + run: pip install --user pipenv - name: Cache Python dependencies - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: .venv - key: pip-3.9-${{ hashFiles('**/Pipfile.lock') }} + key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} restore-keys: | - pip-3.9- + ${{ env.PYTHON_VERSION }}- pip- - name: Install dependencies run: pipenv sync env: - PIPENV_DEFAULT_PYTHON_VERSION: 3.9 + PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - name: Run scrapers run: | @@ -64,3 +57,9 @@ jobs: run: | export PYTHONPATH=$(pwd):$PYTHONPATH pipenv run scrapy combinefeeds -s LOG_ENABLED=False + + - name: Prevent workflow deactivation + uses: gautamkrishnar/keepalive-workflow@v1 + with: + committer_username: "citybureau-bot" + committer_email: "documenters@citybureau.org"