diff --git a/.github/workflows/archive.yml b/.github/workflows/archive.yml index a512110..022b40a 100644 --- a/.github/workflows/archive.yml +++ b/.github/workflows/archive.yml @@ -12,6 +12,7 @@ env: AUTOTHROTTLE_MAX_DELAY: 30.0 AUTOTHROTTLE_START_DELAY: 1.5 AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0 + PYTHON_VERSION: '3.9' jobs: crawl: @@ -19,27 +20,27 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: ${{ env.PYTHON_VERSION }} # Use the constant here - name: Install Pipenv - uses: dschep/install-pipenv-action@v1 + run: pip install --user pipenv - name: Cache Python dependencies uses: actions/cache@v1 with: path: .venv - key: pip-3.8-${{ hashFiles('**/Pipfile.lock') }} + key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} # Use the constant here restore-keys: | - pip-3.8- + pip-${{ env.PYTHON_VERSION }}- pip- - name: Install dependencies run: pipenv sync env: - PIPENV_DEFAULT_PYTHON_VERSION: 3.8 + PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} # Use the constant here - name: Run scrapers run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba05240..c430ed5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,14 +28,14 @@ jobs: - name: Install Pipenv uses: dschep/install-pipenv-action@v1 - # - name: Cache Python dependencies - # uses: actions/cache@v1 - # with: - # path: .venv - # key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }} - # restore-keys: | - # pip-${{ matrix.python-version }}- - # pip- + - name: Cache Python dependencies + uses: actions/cache@v1 + with: + path: .venv + key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }} + restore-keys: | + pip-${{ matrix.python-version }}- + pip- - name: Install dependencies run: pipenv sync --dev @@ -56,7 +56,7 @@ jobs: - name: Test with pytest run: | - pipenv run pytest + pipenv run pytest || [ $? -eq 5 ] - name: Validate output with scrapy if: github.event_name == 'pull_request' @@ -66,3 +66,9 @@ jobs: git diff-index --name-only --diff-filter=d $(git merge-base HEAD ${{ github.base_ref }}) | \ grep -Pio '(?<=/spiders/).*(?=\.py)' | \ xargs pipenv run scrapy validate + + - name: Prevent workflow deactivation + uses: gautamkrishnar/keepalive-workflow@v1 + with: + committer_username: "citybureau-bot" + committer_email: "documenters@citybureau.org" diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 91bcbe8..2ccb37e 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -2,58 +2,61 @@ name: Cron on: schedule: - # Set any time that you'd like scrapers to run (in UTC) - - cron: "29 6 * * *" + - cron: "27 6 * * *" workflow_dispatch: env: CI: true PIPENV_VENV_IN_PROJECT: true SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod - WAYBACK_ENABLED: true AUTOTHROTTLE_MAX_DELAY: 30.0 AUTOTHROTTLE_START_DELAY: 1.5 AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0 - # Add secrets for the platform you're using and uncomment here - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # S3_BUCKET: ${{ secrets.S3_BUCKET }} AZURE_ACCOUNT_KEY: ${{ secrets.AZURE_ACCOUNT_KEY }} AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }} AZURE_CONTAINER: ${{ secrets.AZURE_CONTAINER }} AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }} - # GOOGLE_APPLICATION_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - # GCS_BUCKET = os.getenv("GCS_BUCKET") - # Setup Sentry, add the DSN to secrets and uncomment here - # SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + OPENVPN_USER: ${{ secrets.OPENVPN_USER }} + OPENVPN_PASS: ${{ secrets.OPENVPN_PASS }} + OPENVPN_CONFIG: ${{ secrets.OPENVPN_CONFIG }} + PYTHON_VERSION: '3.9' jobs: crawl: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v1 - - name: Set up Python 3.9 + - name: Connect to OpenVPN + run: | + sudo apt-get install -y openvpn + echo "$OPENVPN_USER" | sudo tee -a /etc/openvpn/client/auth + echo "$OPENVPN_PASS" | sudo tee -a /etc/openvpn/client/auth + echo "$OPENVPN_CONFIG" | sudo tee -a /etc/openvpn/ovpn.conf + sudo openvpn --config /etc/openvpn/ovpn.conf --daemon + sleep 120 + + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v1 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - name: Install Pipenv - uses: dschep/install-pipenv-action@v1 + run: pip install --user pipenv - name: Cache Python dependencies uses: actions/cache@v1 with: path: .venv - key: pip-3.9-${{ hashFiles('**/Pipfile.lock') }} - restore-keys: | - pip-3.9- + key: pip-${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }} + pip-${{ env.PYTHON_VERSION }}- pip- - name: Install dependencies run: pipenv sync env: - PIPENV_DEFAULT_PYTHON_VERSION: 3.9 + PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - name: Run scrapers run: | @@ -63,4 +66,5 @@ jobs: - name: Combine output feeds run: | export PYTHONPATH=$(pwd):$PYTHONPATH - pipenv run scrapy combinefeeds -s LOG_ENABLED=False + pipenv run scrapy combinefeeds -s LOG_ENABLED=True + AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}