Added bigquery support

.github/dependabot.yml

@@ -83,7 +83,144 @@ updates:
       - dependency-name: com.squareup.okhttp3:okhttp
         versions:
           - 4.9.1
+
   - package-ecosystem: pip
     directory: "/python-sdk"
     schedule:
       interval: daily
+
+  - package-ecosystem: maven
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    target-branch: "2.5.x"
+    ignore:
+      - dependency-name: com.squareup.wire:wire-schema
+        versions:
+          - ">= 3.6.a, < 3.7"
+      - dependency-name: com.squareup.wire:wire-compiler
+        versions:
+          - ">= 3.6.a, < 3.7"
+      - dependency-name: com.ibm.icu:icu4j
+        versions:
+          - ">= 69.1, < 70.1"
+      - dependency-name: org.apache.kafka:connect-api
+        versions:
+          - "> 2.2.1, < 2.3"
+      - dependency-name: org.apache.kafka:connect-api
+        versions:
+          - ">= 2.3.a, < 2.4"
+      - dependency-name: org.apache.kafka:connect-api
+        versions:
+          - ">= 5.a, < 6"
+      - dependency-name: org.apache.kafka:connect-api
+        versions:
+          - ">= 6.1.a, < 6.2"
+      - dependency-name: org.apache.kafka:connect-json
+        versions:
+          - ">= 6.1.a, < 6.2"
+      - dependency-name: org.apache.kafka:connect-runtime
+        versions:
+          - ">= 6.1.a, < 6.2"
+      - dependency-name: org.apache.kafka:kafka_2.12
+        versions:
+          - "> 2.3.0"
+      - dependency-name: org.apache.kafka:kafka-clients
+        versions:
+          - "> 2.3.0"
+      - dependency-name: io.quarkus:quarkus-maven-plugin
+        versions:
+          - 1.11.0.Final
+          - 1.11.1.Final
+          - 1.11.2.Final
+          - 1.12.0.Final
+          - 1.13.0.Final
+      - dependency-name: io.quarkus:quarkus-test-common
+        versions:
+          - 1.11.0.Final
+          - 1.11.1.Final
+          - 1.11.2.Final
+          - 1.12.0.Final
+          - 1.13.0.Final
+      - dependency-name: io.quarkus:quarkus-bom
+        versions:
+          - 1.11.0.Final
+          - 1.11.1.Final
+          - 1.11.2.Final
+          - 1.12.0.Final
+          - 1.13.0.Final
+      - dependency-name: com.networknt:json-schema-validator
+        versions:
+          - 1.0.51
+      - dependency-name: io.confluent:kafka-connect-avro-converter
+        versions:
+          - 6.1.0
+      - dependency-name: io.confluent:kafka-json-schema-serializer
+        versions:
+          - 6.1.0
+      - dependency-name: io.confluent:kafka-protobuf-serializer
+        versions:
+          - 6.1.0
+      - dependency-name: io.confluent:kafka-avro-serializer
+        versions:
+          - 6.1.0
+      - dependency-name: org.infinispan:infinispan-commons
+        versions:
+          - 12.1.0.Dev01
+      - dependency-name: org.infinispan:infinispan-core
+        versions:
+          - 12.1.0.Dev01
+      - dependency-name: org.apache.kafka:connect-json
+        versions:
+          - 6.0.1-ce
+          - 6.0.2-ce
+      - dependency-name: org.apache.kafka:connect-api
+        versions:
+          - 6.0.1-ce
+          - 6.0.2-ce
+      - dependency-name: com.fasterxml.jackson.datatype:jackson-datatype-json-org
+        versions:
+          - 2.12.2
+      - dependency-name: io.grpc:grpc-netty-shaded
+        versions:
+          - 1.36.0
+      - dependency-name: io.grpc:grpc-stub
+        versions:
+          - 1.36.0
+      - dependency-name: io.grpc:grpc-protobuf
+        versions:
+          - 1.36.0
+      - dependency-name: io.grpc:grpc-context
+        versions:
+          - 1.36.0
+      - dependency-name: io.grpc:grpc-core
+        versions:
+          - 1.36.0
+      - dependency-name: io.grpc:grpc-api
+        versions:
+          - 1.36.0
+      - dependency-name: org.scala-lang:scala-library
+        versions:
+          - 2.13.5
+      - dependency-name: org.infinispan:infinispan-client-rest
+        versions:
+          - 12.1.0.Dev01
+      - dependency-name: com.google.protobuf:protobuf-java-util
+        versions:
+          - 3.15.0
+      - dependency-name: com.google.protobuf:protobuf-java
+        versions:
+          - 3.15.0
+      - dependency-name: org.apache.kafka:connect-runtime
+        versions:
+          - 6.0.1-ce
+      - dependency-name: com.squareup.wire:wire-schema
+        versions:
+          - 3.5.0
+      - dependency-name: org.junit.jupiter:junit-jupiter-engine
+        versions:
+          - 5.7.1
+      - dependency-name: com.squareup.okhttp3:okhttp
+        versions:
+          - 4.9.1

.github/scripts/build-and-push-multiarch-images.sh

@@ -13,9 +13,14 @@ RELEASE_TYPE="$4"      # Either 'snapshot' or 'release' or 'pre-release'
 RELEASE_VERSION=${5:-$defvalue}   # Release version (Pass the release version if you also want images tagged with the release version)
 
 # Check if variant type is valid
-if [[ ($VARIANT != "mem-multiarch-images") &&  ($VARIANT != "sql-multiarch-images") &&  ($VARIANT != "kafkasql-multiarch-images") && ($VARIANT != "multiarch-registry-images") ]]
+if [[ ($VARIANT != "mem-multiarch-images") && \
+      ($VARIANT != "sql-multiarch-images") && \
+      ($VARIANT != "kafkasql-multiarch-images") && \
+      ($VARIANT != "gitops-multiarch-images") && \
+      ($VARIANT != "multiarch-registry-images") ]]
 then
-    echo "ERROR: Illegal value '${VARIANT}' for variable '$VARIANT'. Values can only be [mem-multiarch-images, sql-multiarch-images, kafkasql-multiarch-images, multiarch-registry-images]"
+    echo -n "ERROR: Illegal value '${VARIANT}' for variable '$VARIANT'. Values can only be "
+    echo    "[mem-multiarch-images, sql-multiarch-images, kafkasql-multiarch-images, gitops-multiarch-images, multiarch-registry-images]"
     exit 1
 fi
 

.github/workflows/integration-tests.yaml

@@ -115,13 +115,6 @@ jobs:
       - name: Run Integration Tests - sql - migration
         run: make REGISTRY_IMAGE='-Dregistry-sql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-sql:1d' run-sql-migration-integration-tests
 
-      - name: Run Integration Tests - sql - db upgrade
-        run: make REGISTRY_IMAGE='-Dregistry-sql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-sql:1d' run-sql-upgrade-tests
-
-      - name: Run Integration Tests - sql - multitenancy
-        run: make REGISTRY_IMAGE='-Dregistry-sql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-sql:1d' run-sql-multitenancy-integration-tests
-
-
       - name: Collect logs
         if: failure()
         run: ./.github/scripts/collect_logs.sh
@@ -164,9 +157,6 @@ jobs:
       - name: Run Integration Tests - KafkaSql - Migration
         run: make REGISTRY_IMAGE='-Dregistry-kafkasql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-kafkasql:1d' run-kafkasql-migration-integration-tests
 
-      - name: Run Integration Tests - KafkaSql - DBUpgrade
-        run: make REGISTRY_IMAGE='-Dregistry-kafkasql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-kafkasql:1d' run-kafkasql-upgrade-tests
-
       - name: Run Integration Tests - KafkaSql - Auth
         run: make REGISTRY_IMAGE='-Dregistry-kafkasql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-kafkasql:1d' run-kafkasql-auth-tests
 

.github/workflows/registry-rhbq-build.yaml

@@ -73,9 +73,6 @@ jobs:
         - name: Set test profile to all
           run: echo "test_profile=all" >> $GITHUB_ENV
 
-        - name: Prepare Tenant Manager
-          run: make build-integration-tests-multitenancy
-
         - name: Build integration-tests-common
           if: github.event.inputs.skip-tests == 'false'
           run: cd registry && ./mvnw install -Pintegration-tests -pl integration-tests/integration-tests-common

.github/workflows/release-images.yaml

@@ -25,6 +25,25 @@ jobs:
     env:
       RELEASE_TYPE: release
     steps:
+
+      - name: View Disk Usage
+        run: df -h
+
+      # https://github.com/marketplace/actions/free-disk-space-ubuntu
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@76866dbe54312617f00798d1762df7f43def6e5c
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          swap-storage: true
+          large-packages: false
+          # this might remove tools that are actually needed, but frees about 6 GB
+          tool-cache: false
+
+      - name: View Disk Usage
+        run: df -h
+
         # Open-Source Machine emulator that allows you to emulate multiple CPU architectures on your machine
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
@@ -97,4 +116,4 @@ jobs:
         with:
           name: ${{ github.job }}
           url: ${{ secrets.GOOGLE_CHAT_WEBHOOK }}
-          status: ${{ job.status }}
+          status: ${{ job.status }}

.github/workflows/verify.yaml

@@ -7,14 +7,14 @@ on:
       - 'README*'
       - 'docs/**'
       - '.github/workflows/**'
-    branches: [main, '[1-9].[0-9].x']
+    branches: [ main, '[1-9].[0-9].x' ]
   pull_request:
     paths-ignore:
       - '.gitignore'
       - 'LICENSE'
       - 'README*'
       - 'docs/**'
-    branches: [main, '[1-9].[0-9].x']
+    branches: [ main, '[1-9].[0-9].x' ]
 
 concurrency:
   # Only run once for latest commit per ref and cancel other (previous) runs.
@@ -198,10 +198,69 @@ jobs:
         if: github.event_name == 'push'
         run: docker login -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" quay.io
 
-      - name: Build and Push In Memory Multi-arch Images to Quay.io
+      - name: Build and Push KafkaSQL Multi-arch Images to Quay.io
         if: github.event_name == 'push'
         run: ./.github/scripts/build-and-push-multiarch-images.sh ${GITHUB_REF#refs/heads/} quay.io kafkasql-multiarch-images snapshot
 
+  build-verify-gitops:
+    name: Verify GitOps Build
+    runs-on: ubuntu-20.04
+    if: github.repository_owner == 'Apicurio' && !contains(github.event.*.labels.*.name, 'DO NOT MERGE')
+    steps:
+      - name: Checkout Code with Ref '${{ github.ref }}'
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: 'temurin'
+          cache: 'maven'
+
+      # Open-Source Machine emulator that allows you to emulate multiple CPU architectures on your machine
+      - name: Set up QEMU
+        if: github.event_name == 'push'
+        uses: docker/setup-qemu-action@v1
+
+      # Docker CLI plugin for extended build capabilities with BuildKit
+      - name: Set up Docker Buildx
+        if: github.event_name == 'push'
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Inspect builder
+        if: github.event_name == 'push'
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      - name: Build and Test GitOps Variant
+        if: github.ref == 'refs/heads/main'
+        run: make SKIP_TESTS=false BUILD_FLAGS='-DskipCommitIdPlugin=false -Dmaven.wagon.httpconnectionManager.maxTotal=30 -Dmaven.wagon.http.retryHandler.count=5 --no-transfer-progress' build-gitops
+
+      - name: Build and Test GitOps Variant Without UI
+        if: github.ref != 'refs/heads/main'
+        run: make SKIP_TESTS=false BUILD_FLAGS='-DskipUiBuild -Dmaven.wagon.httpconnectionManager.maxTotal=30 -Dmaven.wagon.http.retryHandler.count=5 --no-transfer-progress' build-gitops
+
+      - name: Login to DockerHub Registry
+        if: github.event_name == 'push'
+        run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
+
+      - name: Build and Push GitOps Multi-arch Images to Docker.io
+        if: github.event_name == 'push'
+        run: ./.github/scripts/build-and-push-multiarch-images.sh ${GITHUB_REF#refs/heads/} docker.io gitops-multiarch-images snapshot
+
+      - name: Login to Quay.io Registry
+        if: github.event_name == 'push'
+        run: docker login -u "${{ secrets.QUAY_USERNAME }}" -p "${{ secrets.QUAY_PASSWORD }}" quay.io
+
+      - name: Build and Push GitOps Multi-arch Images to Quay.io
+        if: github.event_name == 'push'
+        run: ./.github/scripts/build-and-push-multiarch-images.sh ${GITHUB_REF#refs/heads/} quay.io gitops-multiarch-images snapshot
+
   build-mem-native-images:
     name: Build and Test In Memory native images
     runs-on: ubuntu-20.04
@@ -356,7 +415,7 @@ jobs:
 
       - name: Build Native executables without UI
         env:
-           SKIP_TESTS: "true"
+          SKIP_TESTS: "true"
         if: github.ref != 'refs/heads/main'
         run: make BUILD_FLAGS='-DskipUiBuild' build-sql-native
 
@@ -387,9 +446,6 @@ jobs:
       - name: Run Integration Tests - sql migration
         run: make REGISTRY_IMAGE='-Dregistry-sql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-sql-native:1d' run-sql-migration-integration-tests
 
-      - name: Run Integration Tests - sql multitenancy
-        run: make REGISTRY_IMAGE='-Dregistry-sql-image=ttl.sh/${{ github.sha }}/apicurio/apicurio-registry-sql-native:1d' run-sql-multitenancy-integration-tests
-
       - name: Collect logs
         if: failure()
         run: ./.github/scripts/collect_logs.sh
@@ -427,7 +483,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        language: [go, js, python]
+        language: [ go, js, python ]
     steps:
       - uses: actions/checkout@v3
 
@@ -443,28 +499,28 @@ jobs:
         if: steps.changes.outputs.openapi == 'true'
         uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4
         with:
-           token: ${{ secrets.ACCESS_TOKEN }}
-           repository: Apicurio/apicurio-registry-client-sdk-${{ matrix.language }}
-           event-type: on-oas-updated
-           client-payload: '{"openapi_file_path": "app/src/main/resources-unfiltered/META-INF/resources/api-specifications/registry/v2/openapi.json"}'
+          token: ${{ secrets.ACCESS_TOKEN }}
+          repository: Apicurio/apicurio-registry-client-sdk-${{ matrix.language }}
+          event-type: on-oas-updated
+          client-payload: '{"openapi_file_path": "app/src/main/resources-unfiltered/META-INF/resources/api-specifications/registry/v2/openapi.json"}'
 
   trigger-examples-build:
     if: github.repository_owner == 'Apicurio' && github.event_name == 'push' && github.ref == 'refs/heads/main' && !contains(github.event.*.labels.*.name, 'DO NOT MERGE')
     runs-on: ubuntu-20.04
-    needs: [build-verify]
+    needs: [ build-verify ]
     steps:
       - name: Repository Dispatch
         uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4
         with:
-           token: ${{ secrets.ACCESS_TOKEN }}
-           repository: Apicurio/apicurio-registry-examples
-           event-type: on-registry-updated
+          token: ${{ secrets.ACCESS_TOKEN }}
+          repository: Apicurio/apicurio-registry-examples
+          event-type: on-registry-updated
 
   # Triggers apicurio-registry-mt-ui build whenever apicurio-registry/ui changes
   trigger-mt-ui-build:
     if: github.repository_owner == 'Apicurio' && github.event_name == 'push' && github.ref == 'refs/heads/main' && !contains(github.event.*.labels.*.name, 'DO NOT MERGE')
     runs-on: ubuntu-20.04
-    needs: [build-verify]
+    needs: [ build-verify ]
     steps:
       - uses: actions/checkout@v3
 
@@ -481,6 +537,6 @@ jobs:
         if: steps.changes.outputs.registry-ui == 'true'
         uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4
         with:
-           token: ${{ secrets.ACCESS_TOKEN }}
-           repository: Apicurio/apicurio-registry-mt-ui
-           event-type: on-registry-ui-update
+          token: ${{ secrets.ACCESS_TOKEN }}
+          repository: Apicurio/apicurio-registry-mt-ui
+          event-type: on-registry-ui-update