Skip to content

Run Dataflow Flex Template #4

Run Dataflow Flex Template

Run Dataflow Flex Template #4

name: Run Dataflow Flex Template
env:
PROJECT_ID: gb-poc-373711
LOCATION: europe-west1
CI_SERVICE_NAME: github-actions
METADATA_TEMPLATE_FILE_PATH: "gs://mazlum_dev/dataflow/templates/team_league/java/team-league-java"
JOB_NAME: "team-league-java"
TEMP_LOCATION: "gs://mazlum_dev/dataflow/temp"
STAGING_LOCATION: "gs://mazlum_dev/dataflow/staging"
SA_EMAIL: "sa-dataflow-dev@gb-poc-373711.iam.gserviceaccount.com"
INPUT_FILE: "gs://mazlum_dev/team_league/input/json/input_teams_stats_raw.json"
SIDE_INPUT_FILE: "gs://mazlum_dev/team_league/input/json/input_team_slogans.json"
TEAM_LEAGUE_DATASET: "mazlum_test"
TEAM_STATS_TABLE: "team_stat"
JOB_TYPE: "team_league_java_ingestion_job"
FAILURE_OUTPUT_DATASET: "mazlum_test"
FAILURE_OUTPUT_TABLE: "job_failure"
FAILURE_FEATURE_NAME: "team_league"
WORKLOAD_IDENTITY_PROVIDER: 'projects/975119474255/locations/global/workloadIdentityPools/gb-github-actions-ci-cd-pool/providers/gb-github-actions-ci-cd-provider'
SA_CI_CD_EMAIL: 'sa-dataflow-dev@gb-poc-373711.iam.gserviceaccount.com'
on:
workflow_dispatch:
jobs:
run-template:
runs-on: ubuntu-latest
permissions:
contents: 'read'
id-token: 'write'
steps:
- name: 'Checkout'
uses: 'actions/checkout@v3'
- name: 'Google auth'
id: 'auth'
uses: 'google-github-actions/auth@v1'
with:
workload_identity_provider: '${{ env.WORKLOAD_IDENTITY_PROVIDER }}'
service_account: '${{ env.SA_CI_CD_EMAIL }}'
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v1'
with:
version: '>= 444.0.0'
project_id: '${{ env.PROJECT_ID }}'
- name: 'Run Flex Template And Dataflow Job'
run: |
scripts/run_dataflow_job.sh