scrapers.yml

name: Scrapers
on:
  schedule:
    - cron: '15 9 * * *'
  workflow_dispatch:
    inputs:
      scrapers:
        description: 'Choose the scraper to run'
        default: 'all'
        options:
          - 'linkedin'
          - 'hsoub_academy'
          - 'mostaql_reviews'
          - 'khamsat_reviews'
env:
  GITHUB_PROFILE_TOKEN: ${{ secrets.GH_PROFILE_TOKEN }}
  LINKEDIN_COOKIES: ${{ secrets.LINKEDIN_COOKIES }}
  LINKEDIN_PROFILE_URL: ${{ vars.LINKEDIN_PROFILE_URL }}
  HSOUB_ACADEMY_PROFILE_URL: ${{ vars.HSOUB_ACADEMY_PROFILE_URL }}
  MOSTAQL_REVIEWS_URL: ${{ vars.MOSTAQL_REVIEWS_URL }}
  KHAMSAT_REVIEWS_URL: ${{ vars.KHAMSAT_REVIEWS_URL }}
jobs:
  Scrapers:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10.x'
          cache: 'pip'
      - name: Install dependencies
        run: pip install -r requirements.txt
      - name: Checking cache of Playwright
        uses: actions/cache@v4
        with:
          path: ~/.cache/ms-playwright/
          key: ${{ runner.os }}-playwright
      - name: Install browser
        run: python -m playwright install --with-deps chromium
      - name: Scrap Hsoub Academy profile
        run: python scrapers.py hsoub_academy
        if: ${{ github.event.inputs.scrapers == 'hsoub_academy' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }}
      - name: Scrap Mostaql reviews
        run: python scrapers.py mostaql_reviews
        if: ${{ github.event.inputs.scrapers == 'mostaql_reviews' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }}
      - name: Scrap Khamsat reviews
        run: python scrapers.py khamsat_reviews
        if: ${{ github.event.inputs.scrapers == 'khamsat_reviews' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }}
      - name: Scrap LinkedIn profile
        run: python scrapers.py linkedin
        if: ${{ github.event.inputs.scrapers == 'linkedin' || github.event.inputs.scrapers == 'all' || github.event_name == 'schedule' }}