Merge remote-tracking branch 'fork/master' into jsonExtraIndexArray

apache · Mar 9, 2024 · 12ddc72 · 12ddc72
2 parents d9d319c + 6b123cb
commit 12ddc72
Show file tree

Hide file tree

Showing 301 changed files with 21,626 additions and 3,339 deletions.
diff --git a/.github/workflows/build-multi-arch-pinot-docker-image.yml b/.github/workflows/build-multi-arch-pinot-docker-image.yml
@@ -45,7 +45,7 @@ jobs:
       commit-id: ${{ steps.generate-build-info.outputs.commit-id }}
       tags: ${{ steps.generate-build-info.outputs.tags }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Generate Build Info
         id: generate-build-info
         env:
@@ -68,13 +68,13 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
         with:
           platforms: linux/${{ matrix.arch }}
       - uses: docker/setup-buildx-action@v2
         name: Set up Docker Buildx
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build and push the Docker image
         env:
           DOCKER_FILE_BASE_DIR: "docker/images/pinot"
@@ -95,11 +95,11 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
       - uses: docker/setup-buildx-action@v2
         name: Set up Docker Buildx
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Create Multi-Arch Manifest
         env:
           TAGS: "${{needs.generate-build-info.outputs.tags}}"

diff --git a/.github/workflows/build-pinot-base-docker-image.yml b/.github/workflows/build-pinot-base-docker-image.yml
@@ -36,11 +36,11 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
       - uses: docker/setup-buildx-action@v2
         name: Set up Docker Buildx
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build and push the Docker image
         env:
           OPEN_JDK_DIST: ${{ matrix.openJdkDist }}

diff --git a/.github/workflows/build-pinot-docker-image.yml b/.github/workflows/build-pinot-docker-image.yml
@@ -51,11 +51,11 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
     - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot"

diff --git a/.github/workflows/build-presto-docker-image.yml b/.github/workflows/build-presto-docker-image.yml
@@ -51,11 +51,11 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
     - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot-presto"

diff --git a/.github/workflows/build-superset-docker-image.yml b/.github/workflows/build-superset-docker-image.yml
@@ -47,11 +47,11 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
     - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build and push the Docker image
       env:
         DOCKER_FILE_BASE_DIR: "docker/images/pinot-superset"

diff --git a/.github/workflows/pinot_compatibility_tests.yml b/.github/workflows/pinot_compatibility_tests.yml
@@ -36,12 +36,12 @@ jobs:
         test_suite: [ "compatibility-verifier/sample-test-suite" ]
     name: Pinot Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: 'maven'
       - name: Setup node
         uses: actions/setup-node@v3
@@ -74,7 +74,7 @@ jobs:
         if: always()
         run: |
           zip -1 -r artifacts.zip /tmp/compatibility-verifier/*
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Store compatibility verifier work directory
         if: always()
         with:

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
@@ -49,19 +49,20 @@ jobs:
     runs-on: ubuntu-latest
     name: Pinot Linter Test Set
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: 'maven'
       - name: Linter Test
         env:
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -91,15 +92,15 @@ jobs:
             skip_bytebuffer: true
     name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: ${{ matrix.distribution }}
           cache: 'maven'
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
         with:
@@ -115,6 +116,7 @@ jobs:
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -131,6 +133,7 @@ jobs:
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -139,7 +142,7 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         continue-on-error: true
         timeout-minutes: 5
         with:
@@ -170,15 +173,15 @@ jobs:
             skip_bytebuffer: true
     name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: ${{ matrix.distribution }}
           cache: 'maven'
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
         with:
@@ -193,6 +196,7 @@ jobs:
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -209,6 +213,7 @@ jobs:
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -217,7 +222,7 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         continue-on-error: true
         timeout-minutes: 5
         with:
@@ -234,6 +239,7 @@ jobs:
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -243,7 +249,7 @@ jobs:
         run: .github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
       - name: Upload coverage to Codecov
         if : ${{ matrix.testset == 1 }}
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         continue-on-error: true
         timeout-minutes: 5
         with:
@@ -267,10 +273,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: 'maven'
       - name: Setup node
         uses: actions/setup-node@v3
@@ -283,7 +289,7 @@ jobs:
           npm install -g npm@6.10.0
           npm --version
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
         with:
@@ -299,6 +305,7 @@ jobs:
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
             -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -B -ntp
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -323,15 +330,15 @@ jobs:
             skip_bytebuffer: true
     name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with skip bytebuffers:${{matrix.skip_bytebuffer}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: ${{ matrix.distribution }}
           cache: 'maven'
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}

diff --git a/.github/workflows/pinot_vuln_check.yml b/.github/workflows/pinot_vuln_check.yml
@@ -37,9 +37,9 @@ jobs:
     name: Verify Docker Image
     runs-on: ubuntu-latest
     steps:
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build the Docker image
         env:
           DOCKER_FILE_BASE_DIR: "docker/images/pinot"

diff --git a/.github/workflows/scripts/pr-tests/.pinot_tests_build.sh b/.github/workflows/scripts/pr-tests/.pinot_tests_build.sh
@@ -29,7 +29,7 @@ if [ "$RUN_INTEGRATION_TESTS" != false ]; then
   # Integration Tests
   mvn clean install \
     -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -Dmaven.plugin.appassembler.skip=true \
-    -am -B -T 16 \
+    -am -B -T 16 -ntp \
     -P github-actions,integration-tests \
     -pl 'pinot-integration-tests' || exit 1
 else
@@ -39,7 +39,7 @@ else
   if [ "$RUN_TEST_SET" == "1" ]; then
     mvn clean install \
       -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -Dmaven.plugin.appassembler.skip=true \
-      -am -B -T 16 \
+      -am -B -T 16 -ntp \
       -P github-actions \
       -pl 'pinot-spi' \
       -pl 'pinot-segment-spi' \
@@ -52,7 +52,7 @@ else
   if [ "$RUN_TEST_SET" == "2" ]; then
     mvn clean install \
       -DskipTests -Dcheckstyle.skip -Dspotless.skip -Denforcer.skip -Dlicense.skip -Dmaven.plugin.appassembler.skip=true \
-      -am -B -T 16 \
+      -am -B -T 16 -ntp \
       -P github-actions \
       -pl '!pinot-integration-tests' \
       -pl '!pinot-perf' \

diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
@@ -28,6 +28,6 @@
     <extension>
         <groupId>com.gradle</groupId>
         <artifactId>common-custom-user-data-maven-extension</artifactId>
-        <version>1.12.5</version>
+        <version>1.13</version>
     </extension>
 </extensions>
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -22,6 +22,8 @@
 
 Before you begin to contribute, make sure you have reviewed [Dev Environment Setup](https://docs.pinot.apache.org/developers/developers-and-contributors/code-setup) and [Code Modules and Organization](https://docs.pinot.apache.org/developers/developers-and-contributors/code-modules-and-organization) sections and that you have created your own fork of the pinot source code.
 
+> If you wish to contribute to the UI codebase, follow [this guide](/pinot-controller/src/main/resources/Readme.md) to setup UI locally for development.
+
 ### Create a design document
 
 If your change is relatively minor, you can skip this step. If you are adding new major feature, we suggest that you add a design document and solicit comments from the community before submitting any code.