integration_airflow.yml

name: Test and Check on Pull Request

on:  # yamllint disable-line rule:truthy
  pull_request:
    paths:
      - orchestrate/*
      - orchestrate/**/*

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# This cancels a run if another change is pushed to the same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  airflow:
    name: Pull Request Airflow Tests
    runs-on: ubuntu-latest

    container: datacoves/ci-airflow-dbt-snowflake:3.2

    env:
      AIRBYTE__EXTRACT_LOCATION: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/load
      AIRFLOW__CORE__DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/airflow/dags
      AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 300

    steps:
      - name: Checkout branch
        uses: actions/checkout@v3.5.0
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Test DAG structure integrity (DagBag Loading)
        run: "python /usr/app/load_dagbag.py"

      - name: Test DBT Sources against DAGs' YAML files
        run: "python /usr/app/test_dags.py --dag-loadtime-threshold 1 --check-variable-usage"