fix: fix BigQuery test record size (#86) #493
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main | |
| on: | |
| schedule: | |
| - cron: '0 4 * * 1,3,5' | |
| push: | |
| branches: | |
| - master | |
| tags: | |
| - v* | |
| pull_request: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| skip-test: | |
| description: 'Skip test' | |
| required: false | |
| type: string | |
| default: "false" | |
| jobs: | |
| check: | |
| name: Check & Publish | |
| runs-on: ubuntu-latest | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Services | |
| - name: Build the docker-compose stack | |
| if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} | |
| run: | | |
| mkdir data | |
| curl -s -o data/mysql.sql https://raw.githubusercontent.com/harryho/db-samples/master/mysql/northwind.sql | |
| curl -s -o data/postgres.sql https://raw.githubusercontent.com/harryho/db-samples/master/pgsql/northwind.sql | |
| curl -s -o data/mongo.jsonl https://raw.githubusercontent.com/ozlerhakan/mongodb-json-files/master/datasets/books.json | |
| curl -s -o data/sqlserver.sql https://raw.githubusercontent.com/microsoft/sql-server-samples/master/samples/databases/northwind-pubs/instnwnd.sql | |
| curl -s -o data/oracle.sql https://raw.githubusercontent.com/oracle-samples/db-sample-schemas/main/customer_orders/co_create.sql | |
| curl -s -o data/oracle_pop.sql https://raw.githubusercontent.com/oracle-samples/db-sample-schemas/main/customer_orders/co_populate.sql | |
| cat data/oracle_pop.sql >> data/oracle.sql | |
| rm data/oracle_pop.sql | |
| docker compose -f docker-compose-ci.yml up -d | |
| sleep 30 | |
| docker compose -f docker-compose-ci.yml exec postgres sh -c "psql -d postgres -U postgres -f /tmp/docker/postgres.sql > /dev/null" | |
| docker compose -f docker-compose-ci.yml exec postgres sh -c "psql -d postgres -U postgres -c 'CREATE DATABASE sync' > /dev/null" | |
| docker compose -f docker-compose-ci.yml exec mongo sh -c "mongoimport --authenticationDatabase admin -c books "mongodb://root:example@localhost/samples" /tmp/docker/mongo.jsonl > /dev/null" | |
| docker compose -f docker-compose-ci.yml exec mysql sh -c "mysql -u root -pmysql_passwd < /tmp/docker/mysql.sql" | |
| docker compose -f docker-compose-ci.yml exec sqlserver sh -c "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P SQLServer_Passwd -C -d msdb -i /tmp/docker/sqlserver.sql > /dev/null" | |
| docker compose -f docker-compose-ci.yml exec oracle sh -c "sqlplus -s system/oracle_passwd@localhost @/tmp/docker/oracle.sql > /dev/null" | |
| # Caches | |
| - name: Gradle cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle*.properties') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| # JDK | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 21 | |
| - name: Validate Gradle wrapper | |
| uses: gradle/actions/wrapper-validation@v4 | |
| # Python | |
| - name: Set up Python | |
| if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} | |
| run: | | |
| python3 -m pip install virtualenv | |
| # Gradle check | |
| - name: Build with Gradle | |
| if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} | |
| env: | |
| GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} | |
| run: | | |
| echo $GOOGLE_SERVICE_ACCOUNT | base64 -d > src/test/resources/gcp-service-account.json | |
| ./gradlew check | |
| # Allure check | |
| - name: Auth to Google Cloud | |
| id: auth | |
| if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 }} | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: '${{ secrets.GOOGLE_SERVICE_ACCOUNT }}' | |
| - uses: rlespinasse/github-slug-action@v5 | |
| - name: Publish allure report | |
| uses: andrcuns/allure-publish-action@v2.8.0 | |
| if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} | |
| env: | |
| GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| JAVA_HOME: /usr/lib/jvm/default-jvm/ | |
| with: | |
| storageType: gcs | |
| resultsGlob: build/allure-results | |
| bucket: internal-kestra-host | |
| baseUrl: "https://internal.kestra.io" | |
| prefix: ${{ format('{0}/{1}', github.repository, 'allure/java') }} | |
| copyLatest: true | |
| ignoreMissingResults: true | |
| # Jacoco | |
| - name: 'Set up Cloud SDK' | |
| if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 }} | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: 'Copy jacoco files' | |
| if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 }} | |
| run: | | |
| gsutil -m rsync -d -r build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}', github.repository, 'jacoco') }} | |
| # Publish | |
| - name: Publish package to Sonatype | |
| if: github.ref == 'refs/heads/master' | |
| env: | |
| ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }} | |
| ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }} | |
| SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }} | |
| SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }} | |
| SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }} | |
| run: | | |
| echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties | |
| echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties | |
| echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties | |
| echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg | |
| ./gradlew publishToSonatype | |
| # Release | |
| - name: Release package to Maven Central | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| env: | |
| ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }} | |
| ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }} | |
| SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }} | |
| SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }} | |
| SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }} | |
| run: | | |
| echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties | |
| echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties | |
| echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties | |
| echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg | |
| ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository | |
| # GitHub Release | |
| - name: Create GitHub release | |
| uses: "marvinpinto/action-automatic-releases@latest" | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| with: | |
| repo_token: "${{ secrets.GITHUB_TOKEN }}" | |
| prerelease: false | |
| files: | | |
| build/libs/*.jar | |
| # Slack | |
| - name: Slack notification | |
| uses: 8398a7/action-slack@v3 | |
| if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }} | |
| with: | |
| status: ${{ job.status }} | |
| job_name: Check & Publish | |
| fields: repo,message,commit,author,action,eventName,ref,workflow,job,took | |
| username: Github Actions | |
| icon_emoji: ':github-actions:' | |
| channel: 'C02DQ1A7JLR' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| - name: Notify failed CI | |
| id: send-ci-failed | |
| if: always() && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && job.status != 'success' | |
| uses: kestra-io/actions/.github/actions/send-ci-failed@main | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} |