Scrapers #360
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrapers | |
| on: | |
| schedule: | |
| - cron: '15 9 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| scrapers: | |
| description: 'Choose the scraper to run' | |
| default: 'all' | |
| options: | |
| - 'linkedin' | |
| - 'hsoub_academy' | |
| - 'mostaql_reviews' | |
| - 'khamsat_reviews' | |
| env: | |
| GITHUB_PROFILE_TOKEN: ${{ secrets.GH_PROFILE_TOKEN }} | |
| LINKEDIN_COOKIES: ${{ secrets.LINKEDIN_COOKIES }} | |
| LINKEDIN_PROFILE_URL: ${{ vars.LINKEDIN_PROFILE_URL }} | |
| HSOUB_ACADEMY_PROFILE_URL: ${{ vars.HSOUB_ACADEMY_PROFILE_URL }} | |
| MOSTAQL_REVIEWS_URL: ${{ vars.MOSTAQL_REVIEWS_URL }} | |
| KHAMSAT_REVIEWS_URL: ${{ vars.KHAMSAT_REVIEWS_URL }} | |
| jobs: | |
| Scrapers: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10.x' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Checking cache of Playwright | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/ms-playwright/ | |
| key: ${{ runner.os }}-playwright | |
| - name: Install browser | |
| run: python -m playwright install --with-deps chromium | |
| - name: Scrap Hsoub Academy profile | |
| run: python scrapers.py hsoub_academy | |
| if: ${{ github.event.inputs.scrapers == 'hsoub_academy' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }} | |
| - name: Scrap Mostaql reviews | |
| run: python scrapers.py mostaql_reviews | |
| if: ${{ github.event.inputs.scrapers == 'mostaql_reviews' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }} | |
| - name: Scrap Khamsat reviews | |
| run: python scrapers.py khamsat_reviews | |
| if: ${{ github.event.inputs.scrapers == 'khamsat_reviews' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }} | |
| - name: Scrap LinkedIn profile | |
| run: python scrapers.py linkedin | |
| if: ${{ github.event.inputs.scrapers == 'linkedin' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }} |