diff --git a/.github/workflows/build-multi-arch-pinot-docker-image.yml b/.github/workflows/build-multi-arch-pinot-docker-image.yml
new file mode 100644
index 00000000000..d6f41d059d0
--- /dev/null
+++ b/.github/workflows/build-multi-arch-pinot-docker-image.yml
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Multi-Arch Platform Docker Image Build and Publish
+
+on:
+ workflow_dispatch:
+ inputs:
+ gitUrl:
+ description: "The Pinot git repo to check out to build, use https."
+ default: "https://github.com/apache/pinot.git"
+ required: true
+ commit:
+ description: "The branch/commit to check out to build Pinot image."
+ default: "master"
+ required: true
+ dockerImageName:
+ description: "The docker image name, default to 'apachepinot/pinot'."
+ default: "apachepinot/pinot"
+ required: true
+ tags:
+ description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+ default: ""
+
+jobs:
+ generate-build-info:
+ name: Generate Build Info
+ runs-on: ubuntu-latest
+ outputs:
+ commit-id: ${{ steps.generate-build-info.outputs.commit-id }}
+ tags: ${{ steps.generate-build-info.outputs.tags }}
+ steps:
+ - uses: actions/checkout@v3
+ - name: Generate Build Info
+ id: generate-build-info
+ env:
+ PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+ PINOT_BRANCH: ${{ github.event.inputs.commit }}
+ TAGS: ${{ github.event.inputs.tags }}
+ run: |
+ .github/workflows/scripts/docker/.pinot_build_info_gen.sh
+ build-pinot-docker-image:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ arch: [ "amd64", "arm64" ]
+ base-image-tag: [ "11-amazoncorretto", "11-ms-openjdk" ]
+ name: Build Pinot Docker Image on ${{ matrix.arch }} with base image ${{ matrix.base-image-tag }}
+ needs: [ generate-build-info ]
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v2
+ name: Set up QEMU
+ with:
+ platforms: linux/${{ matrix.arch }}
+ - uses: docker/setup-buildx-action@v2
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Build and push the Docker image
+ env:
+ DOCKER_FILE_BASE_DIR: "docker/images/pinot"
+ DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+ BUILD_PLATFORM: "linux/${{ matrix.arch }}"
+ BASE_IMAGE_TAG: ${{ matrix.base-image-tag }}
+ PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+ PINOT_BRANCH: "${{needs.generate-build-info.outputs.commit-id}}"
+ TAGS: "${{needs.generate-build-info.outputs.tags}}"
+ run: .github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
+ create-multi-arch-manifest:
+ name: Create Multi-Arch Manifest
+ runs-on: ubuntu-latest
+ needs: [ generate-build-info, build-pinot-docker-image ]
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v2
+ name: Set up QEMU
+ - uses: docker/setup-buildx-action@v2
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Create Multi-Arch Manifest
+ env:
+ TAGS: "${{needs.generate-build-info.outputs.tags}}"
+ BUILD_PLATFORM: "linux/arm64,linux/amd64"
+ BASE_IMAGE_TAGS: "11-amazoncorretto,11-ms-openjdk"
+ run: .github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh
diff --git a/.github/workflows/build-pinot-base-docker-image.yml b/.github/workflows/build-pinot-base-docker-image.yml
new file mode 100644
index 00000000000..b540ffd61ef
--- /dev/null
+++ b/.github/workflows/build-pinot-base-docker-image.yml
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Base Docker Image Build and Publish
+
+on:
+ workflow_dispatch:
+ inputs: { }
+
+jobs:
+ build-pinot-build-docker-image:
+ name: Build Pinot Base Docker Image
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ baseImageType: [ "build", "runtime" ]
+ openJdkDist: [ "amazoncorretto", "ms-openjdk" ]
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v2
+ name: Set up QEMU
+ - uses: docker/setup-buildx-action@v2
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Build and push the Docker image
+ env:
+ OPEN_JDK_DIST: ${{ matrix.openJdkDist }}
+ BASE_IMAGE_TYPE: ${{ matrix.baseImageType }}
+ BUILD_PLATFORM: "linux/amd64,linux/arm64"
+ TAG: "11-${{ matrix.openJdkDist }}"
+ run: .github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
diff --git a/.github/workflows/build-pinot-docker-image.yml b/.github/workflows/build-pinot-docker-image.yml
index 345440750e9..93ab5850a70 100644
--- a/.github/workflows/build-pinot-docker-image.yml
+++ b/.github/workflows/build-pinot-docker-image.yml
@@ -47,13 +47,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
- uses: docker/login-action@v1
+ uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- - uses: docker/setup-qemu-action@v1
+ - uses: docker/setup-qemu-action@v2
name: Set up QEMU
- - uses: docker/setup-buildx-action@v1
+ - uses: docker/setup-buildx-action@v2
name: Set up Docker Buildx
- uses: actions/checkout@v3
- name: Build and push the Docker image
diff --git a/.github/workflows/build-presto-docker-image.yml b/.github/workflows/build-presto-docker-image.yml
index 7ae35253615..43b09ea5def 100644
--- a/.github/workflows/build-presto-docker-image.yml
+++ b/.github/workflows/build-presto-docker-image.yml
@@ -47,13 +47,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
- uses: docker/login-action@v1
+ uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- - uses: docker/setup-qemu-action@v1
+ - uses: docker/setup-qemu-action@v2
name: Set up QEMU
- - uses: docker/setup-buildx-action@v1
+ - uses: docker/setup-buildx-action@v2
name: Set up Docker Buildx
- uses: actions/checkout@v3
- name: Build and push the Docker image
diff --git a/.github/workflows/build-superset-docker-image.yml b/.github/workflows/build-superset-docker-image.yml
index c0e87199169..55d28786252 100644
--- a/.github/workflows/build-superset-docker-image.yml
+++ b/.github/workflows/build-superset-docker-image.yml
@@ -43,13 +43,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
- uses: docker/login-action@v1
+ uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- - uses: docker/setup-qemu-action@v1
+ - uses: docker/setup-qemu-action@v2
name: Set up QEMU
- - uses: docker/setup-buildx-action@v1
+ - uses: docker/setup-buildx-action@v2
name: Set up Docker Buildx
- uses: actions/checkout@v3
- name: Build and push the Docker image
diff --git a/.github/workflows/pinot_compatibility_tests.yml b/.github/workflows/pinot_compatibility_tests.yml
index acb0b1974ca..4059852163f 100644
--- a/.github/workflows/pinot_compatibility_tests.yml
+++ b/.github/workflows/pinot_compatibility_tests.yml
@@ -36,11 +36,23 @@ jobs:
test_suite: [ "compatibility-verifier/sample-test-suite" ]
name: Pinot Compatibility Regression Testing against ${{ github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on ${{ matrix.test_suite }}
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK 11
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v3
with:
java-version: 11
+ distribution: 'adopt'
+ cache: 'maven'
+ - name: Setup node
+ uses: actions/setup-node@v3
+ with:
+ node-version: v10.16.1
+ cache: 'npm'
+ cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+ - name: Install npm
+ run: |
+ npm install -g npm@6.10.0
+ npm --version
- name: Pinot Compatibility Regression Testing
if : ${{github.event_name == 'workflow_dispatch'}}
env:
@@ -50,7 +62,7 @@ jobs:
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -62,7 +74,7 @@ jobs:
if: always()
run: |
zip -1 -r artifacts.zip /tmp/compatibility-verifier/*
- - uses: actions/upload-artifact@v2
+ - uses: actions/upload-artifact@v3
name: Store compatibility verifier work directory
if: always()
with:
diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
index b3331fc7d44..844b7bd6fe8 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -49,16 +49,18 @@ jobs:
runs-on: ubuntu-latest
name: Pinot Linter Test Set
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK 11
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v3
with:
java-version: 11
+ distribution: 'adopt'
+ cache: 'maven'
- name: Linter Test
env:
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -70,15 +72,21 @@ jobs:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
+ # Changed to false in order to improve coverage using unsafe buffers
+ fail-fast: false
matrix:
testset: [ 1, 2 ]
- name: Pinot Unit Test Set ${{ matrix.testset }}
+ java: [ 11, 17, 20 ]
+ distribution: [ "temurin" ]
+ name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
steps:
- - uses: actions/checkout@v2
- - name: Set up JDK 11
- uses: actions/setup-java@v1
+ - uses: actions/checkout@v3
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
with:
- java-version: 11
+ java-version: ${{ matrix.java }}
+ distribution: ${{ matrix.distribution }}
+ cache: 'maven'
# Step that does that actual cache save and restore
- uses: actions/cache@v3
env:
@@ -94,7 +102,7 @@ jobs:
RUN_TEST_SET: ${{ matrix.testset }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -104,8 +112,10 @@ jobs:
run: .github/workflows/scripts/.pinot_test.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
+ continue-on-error: true
+ timeout-minutes: 5
with:
- flags: unittests${{ matrix.testset }}
+ flags: unittests${{ matrix.testset }}${{matrix.distribution}}${{matrix.java}}
name: codecov-unit-tests
fail_ci_if_error: false
verbose: true
@@ -114,15 +124,21 @@ jobs:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
+ # Changed to false in order to improve coverage using unsafe buffers
+ fail-fast: false
matrix:
testset: [ 1, 2 ]
- name: Pinot Integration Test Set ${{ matrix.testset }}
+ java: [ 11, 17, 20 ]
+ distribution: [ "temurin" ]
+ name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}})
steps:
- - uses: actions/checkout@v2
- - name: Set up JDK 11
- uses: actions/setup-java@v1
+ - uses: actions/checkout@v3
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
with:
- java-version: 11
+ java-version: ${{ matrix.java }}
+ distribution: ${{ matrix.distribution }}
+ cache: 'maven'
# Step that does that actual cache save and restore
- uses: actions/cache@v3
env:
@@ -138,7 +154,7 @@ jobs:
RUN_TEST_SET: ${{ matrix.testset }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -148,8 +164,10 @@ jobs:
run: .github/workflows/scripts/.pinot_test.sh
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
+ continue-on-error: true
+ timeout-minutes: 5
with:
- flags: integration${{ matrix.testset }}
+ flags: integration${{ matrix.testset }}${{matrix.distribution}}${{matrix.java}}
name: codecov-integration-tests
fail_ci_if_error: false
verbose: true
@@ -158,16 +176,32 @@ jobs:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
+ # Changed to false in order to improve coverage using unsafe buffers
+ fail-fast: false
matrix:
test_suite: [ "compatibility-verifier/sample-test-suite" ]
- old_commit: [ "release-0.9.0", "master" ]
+ old_commit: [
+ "release-0.9.3", "release-0.10.0", "release-0.11.0", "release-0.12.1", "master"
+ ]
name: Pinot Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
steps:
- uses: actions/checkout@v2
- - name: Set up JDK 1.8
- uses: actions/setup-java@v1
+ - name: Set up JDK 11
+ uses: actions/setup-java@v3
with:
- java-version: 1.8
+ java-version: 11
+ distribution: 'adopt'
+ cache: 'maven'
+ - name: Setup node
+ uses: actions/setup-node@v3
+ with:
+ node-version: v10.16.1
+ cache: 'npm'
+ cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+ - name: Install npm
+ run: |
+ npm install -g npm@6.10.0
+ npm --version
# Step that does that actual cache save and restore
- uses: actions/cache@v3
env:
@@ -184,7 +218,7 @@ jobs:
TEST_SUITE: ${{ matrix.test_suite }}
MAVEN_OPTS: >
-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+ -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-XX:+IgnoreUnrecognizedVMOptions
--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -197,17 +231,20 @@ jobs:
if: github.repository == 'apache/pinot'
runs-on: ubuntu-latest
strategy:
+ # Changed to false in order to improve coverage using unsafe buffers
+ fail-fast: false
matrix:
- # We only test LTS Java versions
- # TODO: add JDK 17 once ready. see: https://github.com/apache/pinot/issues/8529
- java: [ 1.8, 11, 15 ]
+ java: [ 11, 17, 20 ]
+ distribution: [ "temurin" ]
name: Pinot Quickstart on JDK ${{ matrix.java }}
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v3
with:
java-version: ${{ matrix.java }}
+ distribution: ${{ matrix.distribution }}
+ cache: 'maven'
# Step that does that actual cache save and restore
- uses: actions/cache@v3
env:
@@ -225,30 +262,15 @@ jobs:
runs-on: ubuntu-latest
name: Build Presto Pinot Driver
steps:
- - uses: actions/checkout@v2
- - name: Set up JDK 11
- uses: actions/setup-java@v1
- with:
- java-version: 11
- - name: Build presto pinot driver with JDK 11
- env:
- MAVEN_OPTS: >
- -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
- -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
- -XX:+IgnoreUnrecognizedVMOptions
- --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
- --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
- --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
- --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
- --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: |
- mvn clean install -DskipTests -Ppresto-driver -am -B -pl ':presto-pinot-driver' -T 16 || exit 1
+ - uses: actions/checkout@v3
- name: Set up JDK 8
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v3
with:
java-version: 8
+ distribution: 'adopt'
+ cache: 'maven'
- name: Build presto pinot driver with JDK 8
env:
- MAVEN_OPTS: -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+ MAVEN_OPTS: -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
run: |
- mvn clean install -DskipTests -Ppresto-driver -am -B -pl ':presto-pinot-driver' -Djdk.version=8 -T 16 || exit 1
+ mvn clean install -Dmaven.test.skip=true -Ppresto-driver -am -B -pl ':pinot-spi-jdk8',':pinot-common-jdk8',':pinot-segment-spi-jdk8' -Djdk.version=8 -T 16 || exit 1
diff --git a/.github/workflows/pinot_vuln_check.yml b/.github/workflows/pinot_vuln_check.yml
index e471f880490..d9aea49353b 100644
--- a/.github/workflows/pinot_vuln_check.yml
+++ b/.github/workflows/pinot_vuln_check.yml
@@ -37,9 +37,9 @@ jobs:
name: Verify Docker Image
runs-on: ubuntu-latest
steps:
- - uses: docker/setup-qemu-action@v1
+ - uses: docker/setup-qemu-action@v2
name: Set up QEMU
- - uses: docker/setup-buildx-action@v1
+ - uses: docker/setup-buildx-action@v2
name: Set up Docker Buildx
- uses: actions/checkout@v3
- name: Build the Docker image
diff --git a/.github/workflows/scripts/.pinot_compatibility_verifier.sh b/.github/workflows/scripts/.pinot_compatibility_verifier.sh
index 0897eacdd80..93789ce67ca 100755
--- a/.github/workflows/scripts/.pinot_compatibility_verifier.sh
+++ b/.github/workflows/scripts/.pinot_compatibility_verifier.sh
@@ -41,17 +41,49 @@ echo " https://packages.confluent.io/maven/">> ${SETTINGS_FILE}
echo " false">> ${SETTINGS_FILE}
echo " ">> ${SETTINGS_FILE}
echo " ">> ${SETTINGS_FILE}
+
+echo " ">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " central">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " 120000">> ${SETTINGS_FILE}
+echo " 120000">> ${SETTINGS_FILE}
+echo " 3">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+echo " ">> ${SETTINGS_FILE}
+
echo "">> ${SETTINGS_FILE}
# PINOT_MAVEN_OPTS is used to provide additional maven options to the checkoutAndBuild.sh command
export PINOT_MAVEN_OPTS="-s $(pwd)/${SETTINGS_FILE}"
-if [ -z "$newerCommit" ]; then
- echo "Running compatibility regression test against \"${olderCommit}\""
+# Compare commit hash for compatibility verification
+git fetch --all
+NEW_COMMIT_HASH=`git log -1 --pretty=format:'%h' HEAD`
+if [ ! -z "${NEW_COMMIT}" ]; then
+ NEW_COMMIT_HASH=`git log -1 --pretty=format:'%h' ${NEW_COMMIT}`
+fi
+OLD_COMMIT_HASH=`git log -1 --pretty=format:'%h' ${OLD_COMMIT}`
+if [ $? -ne 0 ]; then
+ echo "Failed to get commit hash for commit: \"${OLD_COMMIT}\""
+ OLD_COMMIT_HASH=`git log -1 --pretty=format:'%h' origin/${OLD_COMMIT}`
+fi
+if [ "${NEW_COMMIT_HASH}" == "${OLD_COMMIT_HASH}" ]; then
+ echo "No changes between old commit: \"${OLD_COMMIT}\" and new commit: \"${NEW_COMMIT}\""
+ exit 0
+fi
+
+if [ -z "${NEW_COMMIT}" ]; then
+ echo "Running compatibility regression test against \"${OLD_COMMIT}\""
compatibility-verifier/checkoutAndBuild.sh -w $WORKING_DIR -o $OLD_COMMIT
else
- echo "Running compatibility regression test against \"${olderCommit}\" and \"${newerCommit}\""
- compatibility-verifier/checkoutAndBuild.sh -w $WORKING_DIR -o $OLD_COMMIT -n $NEW_OLD_COMMIT
+ echo "Running compatibility regression test against \"${OLD_COMMIT}\" and \"${NEW_COMMIT}\""
+ compatibility-verifier/checkoutAndBuild.sh -w $WORKING_DIR -o $OLD_COMMIT -n $NEW_COMMIT
fi
compatibility-verifier/compCheck.sh -w $WORKING_DIR -t $TEST_SUITE
diff --git a/.github/workflows/scripts/.pinot_quickstart.sh b/.github/workflows/scripts/.pinot_quickstart.sh
index f27ecc8dcfd..d78f647c0c9 100755
--- a/.github/workflows/scripts/.pinot_quickstart.sh
+++ b/.github/workflows/scripts/.pinot_quickstart.sh
@@ -68,9 +68,9 @@ PASS=0
for i in $(seq 1 2)
do
if [ "$JAVA_VER" -gt 11 ] ; then
- mvn clean install -B -DskipTests=true -Pbin-dist -Dmaven.javadoc.skip=true -Djdk.version=11
+ mvn clean install -B -Dmaven.test.skip=true -Pbin-dist -Dmaven.javadoc.skip=true -Djdk.version=11
else
- mvn clean install -B -DskipTests=true -Pbin-dist -Dmaven.javadoc.skip=true -Djdk.version=${JAVA_VER}
+ mvn clean install -B -Dmaven.test.skip=true -Pbin-dist -Dmaven.javadoc.skip=true -Djdk.version=${JAVA_VER}
fi
if [ $? -eq 0 ]; then
PASS=1
@@ -87,6 +87,7 @@ cd "${DIST_BIN_DIR}"
# Test standalone pinot. Configure JAVA_OPTS for smaller memory, and don't use System.exit
export JAVA_OPTS="-Xms1G -Dlog4j2.configurationFile=conf/log4j2.xml"
+
bin/pinot-admin.sh StartZookeeper &
ZK_PID=$!
sleep 10
@@ -119,6 +120,12 @@ if [ $? -ne 0 ]; then
exit 1
fi
+bin/pinot-admin.sh AddTable -tableConfigFile examples/batch/dimBaseballTeams/dimBaseballTeams_offline_table_config.json -schemaFile examples/batch/dimBaseballTeams/dimBaseballTeams_schema.json -exec
+if [ $? -ne 0 ]; then
+ echo 'Failed to create table dimBaseballTeams.'
+ exit 1
+fi
+
# Ingest Data
d=`pwd`
INSERT_INTO_RES=`curl -X POST --header 'Content-Type: application/json' -d "{\"sql\":\"INSERT INTO baseballStats FROM FILE '${d}/examples/batch/baseballStats/rawdata'\",\"trace\":false}" http://localhost:8099/query/sql`
@@ -128,8 +135,17 @@ if [ $? -ne 0 ]; then
fi
PASS=0
+
+INSERT_INTO_RES=`curl -X POST --header 'Content-Type: application/json' -d "{\"sql\":\"INSERT INTO dimBaseballTeams FROM FILE '${d}/examples/batch/dimBaseballTeams/rawdata'\",\"trace\":false}" http://localhost:8099/query/sql`
+if [ $? -ne 0 ]; then
+ echo 'Failed to ingest data for table baseballStats.'
+ exit 1
+fi
+PASS=0
+
# Wait for 10 Seconds for table to be set up, then query the total count.
sleep 10
+# Validate V1 query count(*) result
for i in $(seq 1 150)
do
QUERY_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from baseballStats limit 1","trace":false}' http://localhost:8099/query/sql`
@@ -143,6 +159,38 @@ do
sleep 2
done
+PASS=0
+
+# Validate V2 query count(*) result
+for i in $(seq 1 150)
+do
+ QUERY_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"SET useMultistageEngine=true; select count(*) from baseballStats limit 1","trace":false}' http://localhost:8099/query/sql`
+ if [ $? -eq 0 ]; then
+ COUNT_STAR_RES=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'`
+ if [[ "${COUNT_STAR_RES}" =~ ^[0-9]+$ ]] && [ "${COUNT_STAR_RES}" -eq 97889 ]; then
+ PASS=1
+ break
+ fi
+ fi
+ sleep 2
+done
+
+PASS=0
+
+# Validate V2 join query results
+for i in $(seq 1 150)
+do
+ QUERY_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"SET useMultistageEngine=true;SELECT a.playerName, a.teamID, b.teamName FROM baseballStats_OFFLINE AS a JOIN dimBaseballTeams_OFFLINE AS b ON a.teamID = b.teamID LIMIT 10","trace":false}' http://localhost:8099/query/sql`
+ if [ $? -eq 0 ]; then
+ RES_0=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'`
+ if [[ "${RES_0}" = "\"David Allan\"" ]]; then
+ PASS=1
+ break
+ fi
+ fi
+ sleep 2
+done
+
cleanup "${PINOT_PID}"
cleanup "${ZK_PID}"
if [ "${PASS}" -eq 0 ]; then
@@ -150,8 +198,8 @@ if [ "${PASS}" -eq 0 ]; then
exit 1
fi
-# Test quick-start-batch
-bin/quick-start-batch.sh &
+# Test quick-start-multi-stage
+bin/pinot-admin.sh QuickStart -type MULTI_STAGE &
PID=$!
# Print the JVM settings
@@ -174,6 +222,38 @@ do
sleep 2
done
+PASS=0
+
+# Validate V2 query count(*) result
+for i in $(seq 1 150)
+do
+ QUERY_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"SET useMultistageEngine=true; select count(*) from baseballStats limit 1","trace":false}' http://localhost:8000/query/sql`
+ if [ $? -eq 0 ]; then
+ COUNT_STAR_RES=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'`
+ if [[ "${COUNT_STAR_RES}" =~ ^[0-9]+$ ]] && [ "${COUNT_STAR_RES}" -eq 97889 ]; then
+ PASS=1
+ break
+ fi
+ fi
+ sleep 2
+done
+
+PASS=0
+
+# Validate V2 join query results
+for i in $(seq 1 150)
+do
+ QUERY_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"SET useMultistageEngine=true;SELECT a.playerName, a.teamID, b.teamName FROM baseballStats_OFFLINE AS a JOIN dimBaseballTeams_OFFLINE AS b ON a.teamID = b.teamID LIMIT 10","trace":false}' http://localhost:8000/query/sql`
+ if [ $? -eq 0 ]; then
+ RES_0=`echo "${QUERY_RES}" | jq '.resultTable.rows[0][0]'`
+ if [[ ${RES_0} = "\"David Allan\"" ]]; then
+ PASS=1
+ break
+ fi
+ fi
+ sleep 2
+done
+
cleanup "${PID}"
if [ "${PASS}" -eq 0 ]; then
echo 'Batch Quickstart failed: Cannot get correct result for count star query.'
diff --git a/.github/workflows/scripts/.pinot_test.sh b/.github/workflows/scripts/.pinot_test.sh
index d88414583c6..1608b6f7e99 100755
--- a/.github/workflows/scripts/.pinot_test.sh
+++ b/.github/workflows/scripts/.pinot_test.sh
@@ -27,18 +27,21 @@ netstat -i
if [ "$RUN_INTEGRATION_TESTS" != false ]; then
# Integration Tests
- mvn clean install -DskipTests -am -B -pl 'pinot-integration-tests' -T 16 || exit 1
+ mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.apply.skip -Dlicense.skip=true -am -B \
+ -pl 'pinot-integration-tests' -T 16 || exit 1
if [ "$RUN_TEST_SET" == "1" ]; then
mvn test -am -B \
-pl 'pinot-integration-tests' \
-Dtest='C*Test,L*Test,M*Test,R*Test,S*Test' \
- -P github-actions,integration-tests-only && exit 0 || exit 1
+ -P github-actions,integration-tests-only \
+ -Dcheckstyle.skip -Dspotless.apply.skip -Dlicense.skip=true && exit 0 || exit 1
fi
if [ "$RUN_TEST_SET" == "2" ]; then
mvn test -am -B \
-pl 'pinot-integration-tests' \
-Dtest='!C*Test,!L*Test,!M*Test,!R*Test,!S*Test' \
- -P github-actions,integration-tests-only && exit 0 || exit 1
+ -P github-actions,integration-tests-only \
+ -Dcheckstyle.skip -Dspotless.apply.skip -Dlicense.skip=true && exit 0 || exit 1
fi
else
# Unit Tests
@@ -60,10 +63,11 @@ else
-pl ':pinot-csv' \
-pl ':pinot-json' \
-pl ':pinot-segment-uploader-default' \
- -P github-actions,no-integration-tests && exit 0 || exit 1
+ -P github-actions,no-integration-tests \
+ -Dcheckstyle.skip -Dspotless.apply.skip -Dlicense.skip=true && exit 0 || exit 1
fi
if [ "$RUN_TEST_SET" == "2" ]; then
- mvn clean install -DskipTests -T 16 || exit 1
+ mvn clean install -DskipTests -Dcheckstyle.skip -Dspotless.apply.skip -Dlicense.skip=true -T 16 || exit 1
mvn test -am -B \
-pl '!pinot-spi' \
-pl '!pinot-segment-spi' \
@@ -79,7 +83,9 @@ else
-pl '!:pinot-csv' \
-pl '!:pinot-json' \
-pl '!:pinot-segment-uploader-default' \
- -P github-actions,no-integration-tests && exit 0 || exit 1
+ -P github-actions,no-integration-tests \
+ -Dspotless.apply.skip -Dcheckstyle.skip -Dspotless.apply.skip -Dlicense.skip=true \
+ && exit 0 || exit 1
fi
fi
diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/src/main/resources/META-INF/services/org.apache.pinot.spi.stream.StreamConsumerFactory b/.github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
old mode 100644
new mode 100755
similarity index 68%
rename from pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/src/main/resources/META-INF/services/org.apache.pinot.spi.stream.StreamConsumerFactory
rename to .github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
index 0bbaa018851..b13b14d786b
--- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-0.9/src/main/resources/META-INF/services/org.apache.pinot.spi.stream.StreamConsumerFactory
+++ b/.github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
@@ -1,3 +1,4 @@
+#!/bin/bash -x
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,4 +17,21 @@
# specific language governing permissions and limitations
# under the License.
#
-org.apache.pinot.plugin.stream.kafka09.KafkaConsumerFactory
\ No newline at end of file
+
+if [ -z "${BUILD_PLATFORM}" ]; then
+ exit 1
+fi
+
+if [ -z "${BASE_IMAGE_TYPE}" ]; then
+ exit 1
+fi
+
+cd docker/images/pinot-base/pinot-base-${BASE_IMAGE_TYPE}
+
+docker buildx build \
+ --no-cache \
+ --platform=${BUILD_PLATFORM} \
+ --file ${OPEN_JDK_DIST}.dockerfile \
+ --tag apachepinot/pinot-base-${BASE_IMAGE_TYPE}:${TAG} \
+ --push \
+ .
diff --git a/.github/workflows/scripts/docker/.pinot_build_info_gen.sh b/.github/workflows/scripts/docker/.pinot_build_info_gen.sh
new file mode 100755
index 00000000000..bca2a0ef8e3
--- /dev/null
+++ b/.github/workflows/scripts/docker/.pinot_build_info_gen.sh
@@ -0,0 +1,41 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+if [ -z "${PINOT_GIT_URL}" ]; then
+ PINOT_GIT_URL="https://github.com/apache/pinot.git"
+fi
+if [ -z "${PINOT_BRANCH}" ]; then
+ PINOT_BRANCH="master"
+fi
+
+# Get pinot commit id
+ROOT_DIR=$(pwd)
+rm -rf /tmp/pinot
+git clone -b ${PINOT_BRANCH} --single-branch ${PINOT_GIT_URL} /tmp/pinot
+cd /tmp/pinot
+COMMIT_ID=$(git rev-parse --short HEAD)
+VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+rm -rf /tmp/pinot
+DATE=$(date +%Y%m%d)
+
+if [ -z "${TAGS}" ]; then
+ TAGS="${VERSION}-${COMMIT_ID}-${DATE},latest"
+fi
+echo "commit-id=${COMMIT_ID}" >>"$GITHUB_OUTPUT"
+echo "tags=${TAGS}" >>"$GITHUB_OUTPUT"
diff --git a/.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh b/.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh
new file mode 100755
index 00000000000..3a2995facfd
--- /dev/null
+++ b/.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh
@@ -0,0 +1,62 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+ DOCKER_IMAGE_NAME="apachepinot/pinot"
+fi
+
+tags=()
+declare -a tags=($(echo ${TAGS} | tr "," " "))
+
+platforms=()
+declare -a platforms=($(echo ${BUILD_PLATFORM} | tr "," " "))
+
+baseImageTags=()
+declare -a baseImageTags=($(echo ${BASE_IMAGE_TAGS} | tr "," " "))
+
+for tag in "${tags[@]}"; do
+ for baseImageTag in "${baseImageTags[@]}"; do
+ DOCKER_AMEND_TAGS_CMD=""
+ for platform in "${platforms[@]}"; do
+ platformTag=${platform/\//-}
+ DOCKER_AMEND_TAGS_CMD+=" --amend ${DOCKER_IMAGE_NAME}:${tag}-${baseImageTag}-${platformTag} "
+ done
+
+ echo "Creating manifest for tag: ${tag}-${baseImageTag}"
+ docker manifest create \
+ ${DOCKER_IMAGE_NAME}:${tag}-${baseImageTag} \
+ ${DOCKER_AMEND_TAGS_CMD}
+
+ docker manifest push \
+ ${DOCKER_IMAGE_NAME}:${tag}-${baseImageTag}
+
+ if [ "${baseImageTag}" == "11-amazoncorretto" ]; then
+ if [ "${tag}" == "latest" ]; then
+ echo "Creating manifest for tag: latest"
+ docker manifest create \
+ ${DOCKER_IMAGE_NAME}:latest \
+ ${DOCKER_AMEND_TAGS_CMD}
+
+ docker manifest push \
+ ${DOCKER_IMAGE_NAME}:latest
+ fi
+ fi
+ done
+done
diff --git a/.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh b/.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
new file mode 100755
index 00000000000..c2f8b288a9e
--- /dev/null
+++ b/.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
@@ -0,0 +1,66 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+ DOCKER_IMAGE_NAME="apachepinot/pinot"
+fi
+if [ -z "${PINOT_GIT_URL}" ]; then
+ PINOT_GIT_URL="https://github.com/apache/pinot.git"
+fi
+if [ -z "${JDK_VERSION}" ]; then
+ JDK_VERSION="11"
+fi
+
+tags=()
+declare -a tags=($(echo ${TAGS} | tr "," " "))
+
+cd ${DOCKER_FILE_BASE_DIR}
+platformTag=${BUILD_PLATFORM/\//-}
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"; do
+ DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag}-${BASE_IMAGE_TAG}-${platformTag} "
+
+ if [ "${BASE_IMAGE_TAG}" == "11-amazoncorretto" ]; then
+ if [ "${tag}" == "latest" ]; then
+ DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:latest-${platformTag} "
+ fi
+ fi
+done
+
+echo "Building docker image for platform: ${BUILD_PLATFORM} with tags: ${DOCKER_BUILD_TAGS}"
+docker build \
+ --no-cache \
+ --platform ${BUILD_PLATFORM} \
+ --file Dockerfile \
+ --build-arg PINOT_GIT_URL=${PINOT_GIT_URL} \
+ --build-arg PINOT_BRANCH=${PINOT_BRANCH} \
+ --build-arg JDK_VERSION=${JDK_VERSION} \
+ ${DOCKER_BUILD_TAGS} \
+ .
+
+for tag in "${tags[@]}"; do
+ docker push ${DOCKER_IMAGE_NAME}:${tag}-${BASE_IMAGE_TAG}-${platformTag}
+
+ if [ "${BASE_IMAGE_TAG}" == "11-amazoncorretto" ]; then
+ if [ "${tag}" == "latest" ]; then
+ docker push ${DOCKER_IMAGE_NAME}:${tag}-${platformTag}
+ fi
+ fi
+done
diff --git a/LICENSE-binary b/LICENSE-binary
index 41b098b628f..e22858ca850 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -270,7 +270,7 @@ com.uber:h3:4.0.0
com.yahoo.datasketches:memory:0.8.3
com.yahoo.datasketches:sketches-core:0.8.3
com.yammer.metrics:metrics-core:2.2.0
-com.yscope.clp:clp-ffi:0.2.1
+com.yscope.clp:clp-ffi:0.4.3
com.zaxxer:HikariCP-java7:2.4.13
commons-cli:commons-cli:1.2
commons-codec:commons-codec:1.15
@@ -459,7 +459,7 @@ org.scala-lang.modules:scala-collection-compat_2.12:2.3.0
org.scala-lang.modules:scala-java8-compat_2.12:0.9.1
org.scala-lang.modules:scala-xml_2.12:1.3.0
org.typelevel:macro-compat_2.12:1.1.1
-org.webjars:swagger-ui:3.23.11
+org.webjars:swagger-ui:5.1.0
org.wildfly.openssl:wildfly-openssl:1.0.7.Final
org.xerial.larray:larray-buffer:0.4.1
org.xerial.larray:larray-mmap:0.4.1
diff --git a/compatibility-verifier/checkoutAndBuild.sh b/compatibility-verifier/checkoutAndBuild.sh
index 702e01b1f4a..0bbb531740f 100755
--- a/compatibility-verifier/checkoutAndBuild.sh
+++ b/compatibility-verifier/checkoutAndBuild.sh
@@ -22,6 +22,8 @@ cmdName=`basename $0`
cmdDir=`dirname $0`
source ${cmdDir}/utils.inc
MVN_CACHE_DIR="mvn-compat-cache"
+df -h
+du -hd2 /home/runner
# get usage of the script
function usage() {
@@ -64,7 +66,7 @@ function checkOut() {
# compatibility tester
# It uses a different version for each build, since we build trees in parallel.
# Building the same version on all trees in parallel causes unstable builds.
-# Using indpendent buildIds will cause maven cache to fill up, so we use a
+# Using independent buildIds will cause maven cache to fill up, so we use a
# dedicated maven cache for these builds, and remove the cache after build is
# completed.
# If buildId is less than 0, then the mvn version is not changed in pom files.
@@ -72,10 +74,13 @@ function build() {
local outFile=$1
local buildTests=$2
local buildId=$3
+ local buildCompatibilityVerifier=$4
local repoOption=""
- local versionOption="-Djdk.version=8"
+ local versionOption="-Djdk.version=11"
+ local maxRetry=5
mkdir -p ${MVN_CACHE_DIR}
+ mkdir -p ${mvnCache}
if [ ${buildId} -gt 0 ]; then
# Build it in a different env under different version so that maven cache does
@@ -85,15 +90,40 @@ function build() {
mvn versions:commit -q -B 1>${outFile} 2>&1
repoOption="-Dmaven.repo.local=${mvnCache}/${buildId}"
fi
-
- mvn install package -DskipTests -Pbin-dist ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1
- if [ $? -ne 0 ]; then exit 1; fi
- mvn -pl pinot-tools package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
- if [ $? -ne 0 ]; then exit 1; fi
+ buildComponents=":pinot-tools"
+ if [ $buildCompatibilityVerifier -gt 0 ]; then
+ buildComponents=":pinot-tools,:pinot-compatibility-verifier"
+ fi
+ for i in $(seq 1 $maxRetry); do
+ mvn clean package -am -pl ${buildComponents} -DskipTests -T1C ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1
+ if [ $? -eq 0 ]; then break; fi
+ if [ $i -eq $maxRetry ]; then exit 1; fi
+ echo ""
+ echo "Build failed, see last 1000 lines of output below."
+ tail -1000 ${outFile}
+ echo "Retrying after 30 seconds..."
+ sleep 30
+ done
if [ $buildTests -eq 1 ]; then
- mvn -pl pinot-integration-tests package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
- if [ $? -ne 0 ]; then exit 1; fi
+ for i in $(seq 1 $maxRetry); do
+ mvn -pl :pinot-integration-tests package -DskipTests -T1C ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
+ if [ $? -eq 0 ]; then break; fi
+ if [ $i -eq $maxRetry ]; then exit 1; fi
+ echo ""
+ echo "Build failed, see last 500 lines of output below."
+ tail -500 ${outFile}
+ echo "Retrying after 30 seconds..."
+ sleep 30
+ done
fi
+ # DELETE the mvn cache
+ du -hd1 ${mvnCache}
+ df -h
+ rm -rf ${mvnCache}
+ du -hd2 /home/runner
+ df -h
+ rm -rf /home/runner/.m2/repository
+ df -h
}
#
@@ -190,29 +220,46 @@ fi
echo "Checking out old version at commit \"${olderCommit}\""
checkOut "$olderCommit" "$oldTargetDir"
-# Start builds in parallel.
+exitStatus=0
+
+# Start builds in sequential.
# First build the current tree. We need it so that we can
-# run the compatibiity tester.
+# run the compatibility tester.
echo Starting build for compat checker at ${cmdDir}, buildId none.
-(cd ${cmdDir}/..; build ${curBuildOutFile} 1 -1) &
+(cd ${cmdDir}/..; build ${curBuildOutFile} 1 -1 1) &
curBuildPid=$!
+echo Awaiting build complete for compat checker
+while true ; do
+ printf "."
+ ps -p ${curBuildPid} 1>/dev/null 2>&1
+ if [ $? -ne 0 ]; then
+ printf "\n"
+ wait ${curBuildPid}
+ curBuildStatus=$?
+ break
+ fi
+ sleep 5
+done
+
+if [ ${curBuildStatus} -eq 0 ]; then
+ echo Compat checker build completed successfully
+else
+ echo Compat checker build failed. See ${curBuildOutFile}
+ echo ======== Build output ========
+ cat ${curBuildOutFile}
+ echo ==== End Build output ========
+ exitStatus=1
+ /bin/rm -r ${mvnCache}
+ exit ${exitStatus}
+fi
+
# The old commit has been cloned in oldTargetDir, build it.
buildId=$(date +%s)
echo Starting build for old version at ${oldTargetDir} buildId ${buildId}
-(cd ${oldTargetDir}; build ${oldBuildOutFile} 0 ${buildId}) &
+(cd ${oldTargetDir}; build ${oldBuildOutFile} 0 ${buildId} 0) &
oldBuildPid=$!
-# In case the user specified the current tree as newer commit, then
-# We don't need to build newer commit tree (we have already built the
-# current tree above and linked newTargetDir). Otherwise, build the newTargetDir
-if [ ${buildNewTarget} -eq 1 ]; then
- buildId=$((buildId+1))
- echo Starting build for new version at ${newTargetDir} buildId ${buildId}
- (cd ${newTargetDir}; build ${newBuildOutFile} 0 ${buildId}) &
- newBuildPid=$!
-fi
-
# We may have potentially started three builds above (at least two).
# Wait for each of them to complete.
echo Awaiting build complete for old commit
@@ -228,18 +275,27 @@ while true ; do
sleep 5
done
-echo Awaiting build complete for compat checker
-while true ; do
- printf "."
- ps -p ${curBuildPid} 1>/dev/null 2>&1
- if [ $? -ne 0 ]; then
- printf "\n"
- wait ${curBuildPid}
- curBuildStatus=$?
- break
- fi
- sleep 5
-done
+if [ ${oldBuildStatus} -eq 0 ]; then
+ echo Old version build completed successfully
+else
+ echo Old version build failed. See ${oldBuildOutFile}
+ echo ======== Build output ========
+ cat ${oldBuildOutFile}
+ echo ==== End Build output ========
+ exitStatus=1
+ /bin/rm -r ${mvnCache}
+ exit ${exitStatus}
+fi
+
+# In case the user specified the current tree as newer commit, then
+# We don't need to build newer commit tree (we have already built the
+# current tree above and linked newTargetDir). Otherwise, build the newTargetDir
+if [ ${buildNewTarget} -eq 1 ]; then
+ buildId=$((buildId+1))
+ echo Starting build for new version at ${newTargetDir} buildId ${buildId}
+ (cd ${newTargetDir}; build ${newBuildOutFile} 0 ${buildId} 0) &
+ newBuildPid=$!
+fi
if [ ${buildNewTarget} -eq 1 ]; then
echo Awaiting build complete for new commit
@@ -256,28 +312,6 @@ if [ ${buildNewTarget} -eq 1 ]; then
done
fi
-exitStatus=0
-
-if [ ${oldBuildStatus} -eq 0 ]; then
- echo Old version build completed successfully
-else
- echo Old version build failed. See ${oldBuildOutFile}
- echo ======== Build output ========
- cat ${oldBuildOutFile}
- echo ==== End Build output ========
- exitStatus=1
-fi
-
-if [ ${curBuildStatus} -eq 0 ]; then
- echo Compat checker build completed successfully
-else
- echo Compat checker build failed. See ${curBuildOutFile}
- echo ======== Build output ========
- cat ${curBuildOutFile}
- echo ==== End Build output ========
- exitStatus=1
-fi
-
if [ ${buildNewTarget} -eq 1 ]; then
if [ ${newBuildStatus} -eq 0 ]; then
echo New version build completed successfully
@@ -287,6 +321,7 @@ if [ ${buildNewTarget} -eq 1 ]; then
cat ${newBuildOutFile}
echo ==== End Build output ========
exitStatus=1
+ exit ${exitStatus}
fi
fi
diff --git a/config/suppressions.xml b/config/suppressions.xml
index 56b06ed6b7a..10d796b2333 100644
--- a/config/suppressions.xml
+++ b/config/suppressions.xml
@@ -45,4 +45,7 @@
+
+
+
diff --git a/contrib/pinot-druid-benchmark/pom.xml b/contrib/pinot-druid-benchmark/pom.xml
index 050e312a1b2..0e4589761e0 100644
--- a/contrib/pinot-druid-benchmark/pom.xml
+++ b/contrib/pinot-druid-benchmark/pom.xml
@@ -33,7 +33,7 @@
org.apache.httpcomponents
httpclient
- 4.5.1
+ 4.5.13
diff --git a/contrib/pinot-fmpp-maven-plugin/pom.xml b/contrib/pinot-fmpp-maven-plugin/pom.xml
index b7ccad0d855..7ed1a2dab60 100644
--- a/contrib/pinot-fmpp-maven-plugin/pom.xml
+++ b/contrib/pinot-fmpp-maven-plugin/pom.xml
@@ -36,9 +36,9 @@
maven-plugin
${basedir}/../..
- 3.3.3
+ 3.8.2
0.9.16
- 2.3.28
+ 2.3.30
@@ -55,17 +55,33 @@
org.codehaus.plexus
plexus-utils
+
+ org.eclipse.sisu
+ org.eclipse.sisu.plexus
+
org.apache.maven
maven-plugin-api
${maven.version}
+
+
+ org.eclipse.sisu
+ org.eclipse.sisu.plexus
+
+
net.sourceforge.fmpp
fmpp
${fmpp.version}
+
+
+ org.freemarker
+ freemarker
+
+
org.freemarker
diff --git a/docker/images/pinot-base/README.md b/docker/images/pinot-base/README.md
index d7f5e929174..92430de06c6 100644
--- a/docker/images/pinot-base/README.md
+++ b/docker/images/pinot-base/README.md
@@ -20,40 +20,35 @@
-->
# docker-pinot-base
+
This is the base docker image to build [Apache Pinot](https://github.com/apache/pinot).
-## How to build a docker image
+## Build and publish the docker image
-Arguments:
+Here is
+the [Github Action task](https://github.com/apachepinot/pinot-fork/actions/workflows/build-pinot-docker-base-image.yml)
+to build and publish pinot base docker images.
-`JAVA_VERSION`: The Java Build and Runtime image version. Default is `11`
+This task can be triggered manually to build the cross platform(amd64 and arm64v8) base image.
-`OPENJDK_IMAGE`: Base image to use for Pinot build and runtime, e.g. `arm64v8/openjdk`. Default is `openjdk`.
+The build shell is:
-Usage:
-```SHELL
-docker build -t apachepinot/pinot-base-build:openjdk11 --no-cache --network=host --build-arg JAVA_VERSION=11 -f pinot-base-build/Dockerfile .
-```
+For Amazon Corretto 11:
```SHELL
-docker build -t apachepinot/pinot-base-runtime:openjdk11 --no-cache --network=host --build-arg JAVA_VERSION=11 -f pinot-base-runtime/Dockerfile .
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-build/amazoncorretto.dockerfile --tag apachepinot/pinot-base-build:11-amazoncorretto --push .
```
-Note that if you are not on arm64 machine, you can still build the image by turning on the experimental feature of docker, and add `--platform linux/arm64` into the `docker build ...` script, e.g.
-```SHELL
-docker build -t apachepinot/pinot-base-build:openjdk11-arm64v8 --platform linux/arm64 --no-cache --network=host --build-arg JAVA_VERSION=11 --build-arg OPENJDK_IMAGE=arm64v8/openjdk -f pinot-base-build/Dockerfile .
-```
```SHELL
-docker build -t apachepinot/pinot-base-runtime:openjdk11-arm64v8 --platform linux/arm64 --no-cache --network=host --build-arg JAVA_VERSION=11 --build-arg OPENJDK_IMAGE=arm64v8/openjdk -f pinot-base-runtime/Dockerfile .
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-runtime/amazoncorretto.dockerfile --tag apachepinot/pinot-base-runtime:11-amazoncorretto --push .
```
-## Publish the docker image
+For MS OpenJDK, the build shell is:
-Here is the [Github Action task](https://github.com/apachepinot/pinot-fork/actions/workflows/build-pinot-docker-base-image.yml) to build and publish pinot base docker images.
-
-This task can be triggered manually to build the cross platform(amd64 and arm64v8) base image.
+```SHELL
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-build/amazoncorretto.dockerfile --tag apachepinot/pinot-base-build:11-ms-openjdk --push .
+```
-The build shell is:
```SHELL
-docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file Dockerfile --tag apachepinot/pinot-base-build:openjdk11 --push .
-```
\ No newline at end of file
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-runtime/amazoncorretto.dockerfile --tag apachepinot/pinot-base-runtime:11-ms-openjdk --push .
+```
diff --git a/docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile b/docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile
new file mode 100644
index 00000000000..674f87d6cbb
--- /dev/null
+++ b/docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile
@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM debian:buster-slim
+
+ARG version=11.0.18.10-1
+# In addition to installing the Amazon corretto, we also install
+# fontconfig. The folks who manage the docker hub's
+# official image library have found that font management
+# is a common usecase, and painpoint, and have
+# recommended that Java images include font support.
+#
+# See:
+# https://github.com/docker-library/official-images/blob/master/test/tests/java-uimanager-font/container.java
+
+LABEL MAINTAINER=dev@pinot.apache.org
+
+RUN set -eux \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends \
+ curl ca-certificates gnupg software-properties-common fontconfig java-common vim wget git automake bison flex g++ libboost-all-dev libevent-dev libssl-dev libtool make pkg-config\
+ && curl -fL https://apt.corretto.aws/corretto.key | apt-key add - \
+ && add-apt-repository 'deb https://apt.corretto.aws stable main' \
+ && mkdir -p /usr/share/man/man1 || true \
+ && apt-get update \
+ && apt-get install -y java-11-amazon-corretto-jdk=1:$version \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV LANG C.UTF-8
+ENV JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto
+
+# install maven
+RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
+ && wget https://dlcdn.apache.org/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz -P /tmp \
+ && tar -xzf /tmp/apache-maven-*.tar.gz -C /usr/share/maven --strip-components=1 \
+ && rm -f /tmp/apache-maven-*.tar.gz \
+ && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+ENV MAVEN_HOME /usr/share/maven
+ENV MAVEN_CONFIG /opt/.m2
+
+# install thrift
+RUN wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /tmp/thrift-0.12.0.tar.gz && \
+ tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \
+ base_dir=`pwd` && \
+ cd /tmp/thrift-0.12.0 && \
+ ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \
+ make install
+
+CMD ["bash"]
diff --git a/docker/images/pinot-base/pinot-base-build/Dockerfile b/docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile
similarity index 65%
rename from docker/images/pinot-base/pinot-base-build/Dockerfile
rename to docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile
index 5b93684c563..7fb80886c7a 100644
--- a/docker/images/pinot-base/pinot-base-build/Dockerfile
+++ b/docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile
@@ -17,20 +17,20 @@
# under the License.
#
ARG JAVA_VERSION=11
-ARG OPENJDK_IMAGE=openjdk
-FROM ${OPENJDK_IMAGE}:${JAVA_VERSION} AS pinot_build_env
+ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu AS pinot_build_env
LABEL MAINTAINER=dev@pinot.apache.org
# extra dependency for running launcher
RUN apt-get update && \
- apt-get install -y --no-install-recommends vim wget curl git automake bison flex g++ libboost-all-dev libevent-dev \
- libssl-dev libtool make pkg-config && \
- rm -rf /var/lib/apt/lists/*
+ apt-get install -y --no-install-recommends vim wget curl git automake bison flex g++ libboost-all-dev libevent-dev \
+ libssl-dev libtool make pkg-config && \
+ rm -rf /var/lib/apt/lists/*
# install maven
RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
- && wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -P /tmp \
+ && wget https://dlcdn.apache.org/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz -P /tmp \
&& tar -xzf /tmp/apache-maven-*.tar.gz -C /usr/share/maven --strip-components=1 \
&& rm -f /tmp/apache-maven-*.tar.gz \
&& ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
@@ -39,10 +39,10 @@ ENV MAVEN_CONFIG /opt/.m2
# install thrift
RUN wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /tmp/thrift-0.12.0.tar.gz && \
- tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \
- base_dir=`pwd` && \
- cd /tmp/thrift-0.12.0 && \
- ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \
- make install
+ tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \
+ base_dir=`pwd` && \
+ cd /tmp/thrift-0.12.0 && \
+ ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \
+ make install
-CMD ["-help"]
+CMD ["bash"]
diff --git a/docker/images/pinot-base/pinot-base-build/openjdk.dockerfile b/docker/images/pinot-base/pinot-base-build/openjdk.dockerfile
new file mode 100644
index 00000000000..0abef1191ff
--- /dev/null
+++ b/docker/images/pinot-base/pinot-base-build/openjdk.dockerfile
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+ARG JAVA_VERSION=21
+ARG JDK_IMAGE=openjdk
+# At 2023-06-14, slim is the only openjdk flavour without medium, high or critical vulns
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-jdk-slim
+
+LABEL MAINTAINER=dev@pinot.apache.org
+
+ENV MAVEN_HOME /usr/share/maven
+ENV MAVEN_CONFIG /root/.m2
+
+# extra dependency for running launcher
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends vim wget curl git automake bison flex g++ libboost-all-dev libevent-dev \
+ libssl-dev libtool make pkg-config && \
+ rm -rf /var/lib/apt/lists/*
+
+# install maven
+RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
+ && wget https://dlcdn.apache.org/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz -P /tmp \
+ && tar -xzf /tmp/apache-maven-*.tar.gz -C /usr/share/maven --strip-components=1 \
+ && rm -f /tmp/apache-maven-*.tar.gz \
+ && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \
+ && mvn help:evaluate -Dexpression=settings.localRepository
+
+
+# install thrift
+RUN wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /tmp/thrift-0.12.0.tar.gz && \
+ tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \
+ base_dir=`pwd` && \
+ cd /tmp/thrift-0.12.0 && \
+ ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \
+ make install
+
+CMD ["bash"]
diff --git a/docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile b/docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile
new file mode 100644
index 00000000000..bb43e2131c2
--- /dev/null
+++ b/docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+ARG JAVA_VERSION=11
+ARG JDK_IMAGE=amazoncorretto
+
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-al2-jdk
+
+LABEL MAINTAINER=dev@pinot.apache.org
+
+RUN yum update -y && \
+ yum groupinstall 'Development Tools' -y && \
+ yum install -y procps vim less wget curl git python sysstat perf libtasn1 && \
+ yum clean all
+
+RUN case `uname -m` in \
+ x86_64) arch=x64; ;; \
+ aarch64) arch=arm64; ;; \
+ *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \
+ esac \
+ && mkdir -p /usr/local/lib/async-profiler \
+ && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.9/async-profiler-2.9-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \
+ && ln -s /usr/local/lib/async-profiler/profiler.sh /usr/local/bin/async-profiler
+
+CMD ["bash"]
diff --git a/docker/images/pinot-base/pinot-base-runtime/Dockerfile b/docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile
similarity index 69%
rename from docker/images/pinot-base/pinot-base-runtime/Dockerfile
rename to docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile
index 7a8ade3016a..364da20d6b4 100644
--- a/docker/images/pinot-base/pinot-base-runtime/Dockerfile
+++ b/docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile
@@ -16,24 +16,25 @@
# specific language governing permissions and limitations
# under the License.
#
+
ARG JAVA_VERSION=11
-ARG OPENJDK_IMAGE=openjdk
+ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk
-FROM ${OPENJDK_IMAGE}:${JAVA_VERSION}-jdk-slim
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu
LABEL MAINTAINER=dev@pinot.apache.org
RUN apt-get update && \
- apt-get install -y --no-install-recommends vim less wget curl git python sysstat procps linux-perf openjdk-11-dbg && \
- rm -rf /var/lib/apt/lists/*
+ apt-get install -y --no-install-recommends vim less wget curl git python sysstat procps linux-tools-generic libtasn1-6 && \
+ rm -rf /var/lib/apt/lists/*
RUN case `uname -m` in \
- x86_64) arch=x64; ;; \
- aarch64) arch=arm64; ;; \
- *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \
+ x86_64) arch=x64; ;; \
+ aarch64) arch=arm64; ;; \
+ *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \
esac \
&& mkdir -p /usr/local/lib/async-profiler \
- && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.5.1/async-profiler-2.5.1-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \
+ && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.9/async-profiler-2.9-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \
&& ln -s /usr/local/lib/async-profiler/profiler.sh /usr/local/bin/async-profiler
CMD ["bash"]
diff --git a/docker/images/pinot-base/pinot-base-runtime/openjdk.dockerfile b/docker/images/pinot-base/pinot-base-runtime/openjdk.dockerfile
new file mode 100644
index 00000000000..e59787749fd
--- /dev/null
+++ b/docker/images/pinot-base/pinot-base-runtime/openjdk.dockerfile
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+ARG JAVA_VERSION=21
+ARG JDK_IMAGE=openjdk
+
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-jdk-slim
+
+LABEL MAINTAINER=dev@pinot.apache.org
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends vim less wget curl git python-is-python3 sysstat procps linux-perf libtasn1-6 && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN case `uname -m` in \
+ x86_64) arch=x64; ;; \
+ aarch64) arch=arm64; ;; \
+ *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \
+ esac \
+ && mkdir -p /usr/local/lib/async-profiler \
+ && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.9/async-profiler-2.9-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \
+ && ln -s /usr/local/lib/async-profiler/profiler.sh /usr/local/bin/async-profiler
+
+CMD ["bash"]
diff --git a/docker/images/pinot-superset/requirements-db.txt b/docker/images/pinot-superset/requirements-db.txt
index bb250db8c99..caf7d4a3ff7 100644
--- a/docker/images/pinot-superset/requirements-db.txt
+++ b/docker/images/pinot-superset/requirements-db.txt
@@ -17,4 +17,4 @@
# under the License.
#
pinotdb>=0.4.5
-redis==3.4.1
+redis==4.5.4
diff --git a/docker/images/pinot/Dockerfile b/docker/images/pinot/Dockerfile
index 499060ed7c7..3378b6f6454 100644
--- a/docker/images/pinot/Dockerfile
+++ b/docker/images/pinot/Dockerfile
@@ -16,16 +16,20 @@
# specific language governing permissions and limitations
# under the License.
#
-ARG PINOT_BASE_IMAGE_TAG=openjdk11
+ARG PINOT_BASE_IMAGE_TAG=11-amazoncorretto
FROM apachepinot/pinot-base-build:${PINOT_BASE_IMAGE_TAG} AS pinot_build_env
LABEL MAINTAINER=dev@pinot.apache.org
ARG PINOT_BRANCH=master
-ARG KAFKA_VERSION=2.0
ARG JDK_VERSION=11
ARG PINOT_GIT_URL="https://github.com/apache/pinot.git"
-RUN echo "Trying to build Pinot from [ ${PINOT_GIT_URL} ] on branch [ ${PINOT_BRANCH} ] with Kafka version [ ${KAFKA_VERSION} ]"
+ARG CI=true
+
+RUN echo "Build Pinot based on image: apachepinot/pinot-base-build:${PINOT_BASE_IMAGE_TAG}"
+RUN echo "Current build system CPU arch is [ $(uname -m) ]"
+
+RUN echo "Trying to build Pinot from [ ${PINOT_GIT_URL} ] on branch [ ${PINOT_BRANCH} ] and CI [ ${CI} ]"
ENV PINOT_HOME=/opt/pinot
ENV PINOT_BUILD_DIR=/opt/pinot-build
ENV MAVEN_HOME /usr/share/maven
@@ -35,6 +39,7 @@ RUN git clone ${PINOT_GIT_URL} ${PINOT_BUILD_DIR} && \
cd ${PINOT_BUILD_DIR} && \
git checkout ${PINOT_BRANCH} && \
mvn install package -DskipTests -Pbin-dist -Pbuild-shaded-jar -Djdk.version=${JDK_VERSION} -T1C && \
+ rm -rf /root/.m2 && \
mkdir -p ${PINOT_HOME}/configs && \
mkdir -p ${PINOT_HOME}/data && \
cp -r build/* ${PINOT_HOME}/. && \
@@ -54,9 +59,8 @@ COPY bin ${PINOT_HOME}/bin
COPY etc ${PINOT_HOME}/etc
COPY examples ${PINOT_HOME}/examples
-RUN wget -O ${PINOT_HOME}/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent-0.12.0.jar https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar
-RUN wget -O ${PINOT_HOME}/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent-0.16.1.jar https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar && \
- ln -s ${PINOT_HOME}/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent-0.16.1.jar ${PINOT_HOME}/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar
+RUN wget -O ${PINOT_HOME}/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent-0.18.0.jar https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.18.0/jmx_prometheus_javaagent-0.18.0.jar && \
+ ln -s ${PINOT_HOME}/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent-0.18.0.jar ${PINOT_HOME}/etc/jmx_prometheus_javaagent/jmx_prometheus_javaagent.jar
# expose ports for controller/broker/server/admin
EXPOSE 9000 8099 8098 8097 8096
diff --git a/docker/images/pinot/etc/conf/pinot-minion-log4j2.xml b/docker/images/pinot/etc/conf/pinot-minion-log4j2.xml
new file mode 100644
index 00000000000..6907f5e9f9d
--- /dev/null
+++ b/docker/images/pinot/etc/conf/pinot-minion-log4j2.xml
@@ -0,0 +1,80 @@
+
+
+
+
+ logs/pinotMinion
+
+
+
+
+ %d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n
+
+
+
+
+ %d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n
+
+
+
+
+
+
+
+
+
+
+ %d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml
index ea9d5e3a859..91e72bc39b5 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/broker.yml
@@ -1,71 +1,71 @@
rules:
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_authorization_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_documentsScanned_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_entriesScannedInFilter_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_entriesScannedPostFilter_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_freshnessLagMs_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queries_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryExecution_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryRouting_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_reduce_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_requestCompilation_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_scatterGather_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_totalServerResponseSize_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_groupBySize_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_noServingHostForSegment_$3"
cache: true
labels:
@@ -98,48 +98,48 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_routingTableUpdateTime_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_brokerResponsesWithPartialServersResponded_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_noServerFoundExceptions_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_brokerResponsesWithProcessingExceptions_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryQuotaExceeded_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryTotalTimeMs_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_serverMissingForRouting_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_deserialization_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_requestConnectionWait_$2"
cache: true
labels:
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
index 5dafc3910c9..88ea8dff920 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/controller.yml
@@ -8,49 +8,49 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_helix_ZookeeperReconnects_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_idealstateZnodeSize_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_idealstateZnodeByteSize_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_replicationFromConfig_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_numberOfReplicas_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_percentOfReplicas_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_percentSegmentsAvailable_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_segmentCount_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_segmentsInErrorState_$3"
cache: true
labels:
@@ -77,49 +77,49 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_offlineTableCount_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_validateion_$2_$3"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_cronSchedulerJobScheduled_$3"
cache: true
labels:
table: "$1"
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
- name: "pinot_controller_cronSchedulerTriggered_$3"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobTriggered_$3"
cache: true
labels:
table: "$1"
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
- name: "pinot_controller_cronSchedulerSkipped_$3"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobSkipped_$3"
cache: true
labels:
table: "$1"
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$3"
cache: true
labels:
table: "$1"
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_taskStatus_$3"
cache: true
labels:
taskType: "$1"
status: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_timeMsSinceLastMinionTaskMetadataUpdate_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
taskType: "$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_$1_$5"
cache: true
labels:
@@ -131,14 +131,14 @@ rules:
cache: true
labels:
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_timeMsSinceLastSuccessfulMinionTaskGeneration_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
taskType: "$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_lastMinionTaskGenerationEncountersError_$4"
cache: true
labels:
@@ -148,47 +148,47 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_pinotLeadControllerResourceEnabled_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_offlineTableEstimatedSize_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableQuota_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_periodicTaskError_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
periodicTask: "$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableStorageQuotaUtilization_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableStorageEstMissingSegmentPercent_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableTotalSizeOnServer_$3"
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableSizePerReplicaOnServer_$3"
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableCompressedSize_$3"
labels:
table: "$1"
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml
index 8d493e90b62..95853f50ecb 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/minion.yml
@@ -4,13 +4,13 @@ rules:
cache: true
labels:
version: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_minion_numberOfTasks_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_minion_$4_$5"
cache: true
labels:
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
index 104d461bbec..a502742e02f 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/pinot.yml
@@ -9,43 +9,43 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_helix_ZookeeperReconnects_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_idealstateZnodeSize_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_idealstateZnodeByteSize_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_numberOfReplicas_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_percentOfReplicas_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_percentSegmentsAvailable_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_segmentCount_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_segmentsInErrorState_$3"
cache: true
labels:
@@ -72,24 +72,30 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_offlineTableCount_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_validateion_$2_$3"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_cronSchedulerJobScheduled_$3"
cache: true
labels:
table: "$1"
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
- name: "pinot_controller_cronSchedulerTriggered_$3"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobTriggered_$3"
cache: true
labels:
table: "$1"
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+ name: "pinot_controller_cronSchedulerJobSkipped_$3"
+ cache: true
+ labels:
+ table: "$1"
+ taskType: "$2"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_cronSchedulerJobExecutionTimeMs_$3"
cache: true
labels:
@@ -101,14 +107,14 @@ rules:
labels:
taskType: "$1"
status: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_timeMsSinceLastMinionTaskMetadataUpdate_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
taskType: "$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_$1_$5"
cache: true
labels:
@@ -120,14 +126,14 @@ rules:
cache: true
labels:
taskType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_timeMsSinceLastSuccessfulMinionTaskGeneration_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
taskType: "$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_lastMinionTaskGenerationEncountersError_$4"
cache: true
labels:
@@ -137,44 +143,44 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_pinotLeadControllerResourceEnabled_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_offlineTableEstimatedSize_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_largestSegmentSizeOnServer_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableTotalSizeOnServer_$3"
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableSizePerReplicaOnServer_$3"
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableCompressedSize_$3"
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableQuota_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableStorageQuotaUtilization_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_controller_tableStorageEstMissingSegmentPercent_$3"
cache: true
labels:
@@ -182,78 +188,78 @@ rules:
tableType: "$2"
# Pinot Broker
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_authorization_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_documentsScanned_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_entriesScannedInFilter_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_entriesScannedPostFilter_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_freshnessLagMs_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queries_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryExecution_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryRouting_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryTotalTimeMs_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_reduce_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_requestCompilation_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_scatterGather_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_totalServerResponseSize_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_groupBySize_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_noServingHostForSegment_$3"
cache: true
labels:
@@ -286,63 +292,63 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_routingTableUpdateTime_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_brokerResponsesWithPartialServersResponded_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_brokerResponsesWithProcessingExceptions_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_brokerResponsesWithNumGroupsLimitReached_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_queryQuotaExceeded_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_serverMissingForRouting_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_deserialization_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_broker_requestConnectionWait_$2"
cache: true
labels:
table: "$1"
# Pinot Server
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_documentCount_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_segmentCount_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_$3_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_realtimeRowsConsumed_$5"
cache: true
labels:
@@ -356,7 +362,7 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_helix_zookeeperReconnects_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_highestKafkaOffsetConsumed_$5"
cache: true
labels:
@@ -364,7 +370,7 @@ rules:
tableType: "$2"
topic: "$3"
partition: "$4"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_highestStreamOffsetConsumed_$5"
cache: true
labels:
@@ -372,7 +378,7 @@ rules:
tableType: "$2"
topic: "$3"
partition: "$4"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_lastRealtimeSegment$1Seconds_$6"
cache: true
labels:
@@ -383,7 +389,7 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_llcControllerResponse_$1_$2"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_llcPartitionConsuming_$5"
cache: true
labels:
@@ -403,7 +409,7 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_realtime_consumptionExceptions_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_realtime_offheapMemoryUsed_$2"
cache: true
labels:
@@ -428,25 +434,25 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_nettyConnection_$1_$2"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_realtimeSegmentNumPartitions_$2"
cache: true
labels:
table: "$1"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_resizeTimeMs_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_upsertPrimaryKeysCount_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
partition: "$3"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_realtimeIngestionDelayMs_$4"
cache: true
labels:
@@ -455,13 +461,13 @@ rules:
partition: "$3"
# Pinot Minions
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_minion_numberOfTasks_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_minion_$4_$5"
cache: true
labels:
diff --git a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml
index 5ad7d15f81c..ad5b96eef84 100644
--- a/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml
+++ b/docker/images/pinot/etc/jmx_prometheus_javaagent/configs/server.yml
@@ -1,23 +1,23 @@
rules:
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_documentCount_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_segmentCount_$3"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_$3_$4"
cache: true
labels:
table: "$1"
tableType: "$2"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_realtimeRowsConsumed_$5"
cache: true
labels:
@@ -31,7 +31,7 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_helix_zookeeperReconnects_$1"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_highestKafkaOffsetConsumed_$5"
cache: true
labels:
@@ -39,7 +39,7 @@ rules:
tableType: "$2"
topic: "$3"
partition: "$4"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_highestStreamOffsetConsumed_$5"
cache: true
labels:
@@ -47,7 +47,7 @@ rules:
tableType: "$2"
topic: "$3"
partition: "$4"
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_lastRealtimeSegment$1Seconds_$6"
cache: true
labels:
@@ -58,7 +58,7 @@ rules:
- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
name: "pinot_server_llcControllerResponse_$1_$2"
cache: true
-- pattern: "\"org.apache.pinot.common.metrics\"<>(\\w+)"
+- pattern: "\"org.apache.pinot.common.metrics\"